diff options
author | Jörg Thalheim <Mic92@users.noreply.github.com> | 2023-04-11 19:49:24 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-04-11 19:49:24 +0100 |
commit | 89103b63d243de631d69404b987a62d38458e329 (patch) | |
tree | b6851cb789a1df3258e506241d7def2bcace0c77 /pkgs/development/tools/continuous-integration | |
parent | 2ed32c48a55086934bc3b1f0427a9d1dc5bbe3b2 (diff) | |
parent | 90b750456ea62408dc4c0bbe68adabdd96e381a7 (diff) |
Merge pull request #225195 from Mic92/buildbot
buildbot: move out of python3.pkgs
Diffstat (limited to 'pkgs/development/tools/continuous-integration')
7 files changed, 457 insertions, 0 deletions
diff --git a/pkgs/development/tools/continuous-integration/buildbot/default.nix b/pkgs/development/tools/continuous-integration/buildbot/default.nix new file mode 100644 index 0000000000000..ec3a4a81f60f0 --- /dev/null +++ b/pkgs/development/tools/continuous-integration/buildbot/default.nix @@ -0,0 +1,42 @@ +{ python3 +, recurseIntoAttrs +, callPackage +}: +let + python = python3.override { + packageOverrides = self: super: { + sqlalchemy = super.sqlalchemy.overridePythonAttrs (oldAttrs: rec { + version = "1.4.40"; + src = super.fetchPypi { + pname = "SQLAlchemy"; + inherit version; + hash = "sha256-RKZgUGCAzJdeHfpXdv5fYxXdxiane1C/Du4YsDieomU="; + }; + }); + moto = super.moto.overridePythonAttrs (oldAttrs: rec { + # a lot of tests -> very slow, we already build them when building python packages + doCheck = false; + }); + }; + }; + + buildbot-pkg = python.pkgs.callPackage ./pkg.nix { + inherit buildbot; + }; + buildbot-worker = python3.pkgs.callPackage ./worker.nix { + inherit buildbot; + }; + buildbot = python.pkgs.callPackage ./master.nix { + inherit buildbot-pkg buildbot-worker buildbot-plugins; + }; + buildbot-plugins = recurseIntoAttrs (callPackage ./plugins.nix { + inherit buildbot-pkg; + }); +in +{ + inherit buildbot buildbot-plugins buildbot-worker; + buildbot-ui = buildbot.withPlugins (with buildbot-plugins; [ www ]); + buildbot-full = buildbot.withPlugins (with buildbot-plugins; [ + www console-view waterfall-view grid-view wsgi-dashboards badges + ]); +} diff --git a/pkgs/development/tools/continuous-integration/buildbot/master.nix b/pkgs/development/tools/continuous-integration/buildbot/master.nix new file mode 100644 index 0000000000000..6e59e31867b62 --- /dev/null +++ b/pkgs/development/tools/continuous-integration/buildbot/master.nix @@ -0,0 +1,154 @@ +{ lib +, stdenv +, buildPythonPackage +, buildPythonApplication +, fetchPypi +, makeWrapper +, pythonOlder +, python +, twisted +, jinja2 +, msgpack +, zope_interface +, sqlalchemy +, alembic +, python-dateutil +, txaio +, autobahn +, pyjwt +, pyyaml +, treq +, txrequests +, pypugjs +, boto3 +, moto +, mock +, lz4 +, setuptoolsTrial +, buildbot-worker +, buildbot-plugins +, buildbot-pkg +, parameterized +, git +, openssh +, glibcLocales +, nixosTests +, callPackage +}: + +let + withPlugins = plugins: buildPythonApplication { + pname = "${package.pname}-with-plugins"; + inherit (package) version; + format = "other"; + + dontUnpack = true; + dontBuild = true; + doCheck = false; + + nativeBuildInputs = [ + makeWrapper + ]; + + propagatedBuildInputs = plugins ++ package.propagatedBuildInputs; + + installPhase = '' + makeWrapper ${package}/bin/buildbot $out/bin/buildbot \ + --prefix PYTHONPATH : "${package}/${python.sitePackages}:$PYTHONPATH" + ln -sfv ${package}/lib $out/lib + ''; + + passthru = package.passthru // { + withPlugins = morePlugins: withPlugins (morePlugins ++ plugins); + }; + }; + + package = buildPythonApplication rec { + pname = "buildbot"; + version = "3.7.0"; + format = "setuptools"; + + disabled = pythonOlder "3.7"; + + src = fetchPypi { + inherit pname version; + hash = "sha256-YMLT1SP6NenJIUVTvr58GVrtNXHw+bhfgMpZu3revG4="; + }; + + propagatedBuildInputs = [ + # core + twisted + jinja2 + msgpack + zope_interface + sqlalchemy + alembic + python-dateutil + txaio + autobahn + pyjwt + pyyaml + ] + # tls + ++ twisted.optional-dependencies.tls; + + nativeCheckInputs = [ + treq + txrequests + pypugjs + boto3 + moto + mock + lz4 + setuptoolsTrial + buildbot-worker + buildbot-pkg + buildbot-plugins.www + parameterized + git + openssh + glibcLocales + ]; + + patches = [ + # This patch disables the test that tries to read /etc/os-release which + # is not accessible in sandboxed builds. + ./skip_test_linux_distro.patch + ]; + + postPatch = '' + substituteInPlace buildbot/scripts/logwatcher.py --replace '/usr/bin/tail' "$(type -P tail)" + ''; + + # Silence the depreciation warning from SqlAlchemy + SQLALCHEMY_SILENCE_UBER_WARNING = 1; + + # TimeoutErrors on slow machines -> aarch64 + doCheck = !stdenv.isAarch64; + + preCheck = '' + export LC_ALL="en_US.UTF-8" + export PATH="$out/bin:$PATH" + + # remove testfile which is missing configuration file from sdist + rm buildbot/test/integration/test_graphql.py + # tests in this file are flaky, see https://github.com/buildbot/buildbot/issues/6776 + rm buildbot/test/integration/test_try_client.py + ''; + + passthru = { + inherit withPlugins; + tests.buildbot = nixosTests.buildbot; + updateScript = ./update.sh; + }; + + meta = with lib; { + description = "An open-source continuous integration framework for automating software build, test, and release processes"; + homepage = "https://buildbot.net/"; + changelog = "https://github.com/buildbot/buildbot/releases/tag/v${version}"; + maintainers = with maintainers; [ ryansydnor lopsided98 ]; + license = licenses.gpl2Only; + broken = stdenv.isDarwin; + }; + }; +in package diff --git a/pkgs/development/tools/continuous-integration/buildbot/pkg.nix b/pkgs/development/tools/continuous-integration/buildbot/pkg.nix new file mode 100644 index 0000000000000..13c0ec8694854 --- /dev/null +++ b/pkgs/development/tools/continuous-integration/buildbot/pkg.nix @@ -0,0 +1,31 @@ +{ lib, buildPythonPackage, fetchPypi, isPy3k, buildbot }: + +buildPythonPackage rec { + pname = "buildbot-pkg"; + inherit (buildbot) version; + + src = fetchPypi { + inherit pname version; + hash = "sha256-4BXCOLW5e7RuZEzyD+oRmS2I4oT2W3oTcH4ZPxwKKvU="; + }; + + postPatch = '' + # Their listdir function filters out `node_modules` folders. + # Do we have to care about that with Nix...? + substituteInPlace buildbot_pkg.py --replace "os.listdir = listdir" "" + ''; + + # No tests + doCheck = false; + + pythonImportsCheck = [ "buildbot_pkg" ]; + + disabled = !isPy3k; + + meta = with lib; { + homepage = "https://buildbot.net/"; + description = "Buildbot Packaging Helper"; + maintainers = with maintainers; [ ryansydnor lopsided98 ]; + license = licenses.gpl2; + }; +} diff --git a/pkgs/development/tools/continuous-integration/buildbot/plugins.nix b/pkgs/development/tools/continuous-integration/buildbot/plugins.nix new file mode 100644 index 0000000000000..42a77fa0f4bc5 --- /dev/null +++ b/pkgs/development/tools/continuous-integration/buildbot/plugins.nix @@ -0,0 +1,141 @@ +{ lib, buildPythonPackage, fetchPypi, callPackage, mock, cairosvg, klein, jinja2, buildbot-pkg }: +{ + www = buildPythonPackage rec { + pname = "buildbot-www"; + inherit (buildbot-pkg) version; + + src = fetchPypi { + inherit pname version; + hash = "sha256-t4xHfox6h5PY4+phdCQbClHd77+WfpUWErMZCEcMxu0="; + }; + + # Remove unnecessary circular dependency on buildbot + postPatch = '' + sed -i "s/'buildbot'//" setup.py + ''; + + buildInputs = [ buildbot-pkg mock ]; + + # No tests + doCheck = false; + + meta = with lib; { + homepage = "https://buildbot.net/"; + description = "Buildbot UI"; + maintainers = with maintainers; [ ryansydnor lopsided98 ]; + license = licenses.gpl2; + }; + }; + + console-view = buildPythonPackage rec { + pname = "buildbot-console-view"; + inherit (buildbot-pkg) version; + + src = fetchPypi { + inherit pname version; + hash = "sha256-JySn7QO+SgoXjVeV4wYwc9twr0Q2c/wsEspqeq038+k="; + }; + + buildInputs = [ buildbot-pkg ]; + + # No tests + doCheck = false; + + meta = with lib; { + homepage = "https://buildbot.net/"; + description = "Buildbot Console View Plugin"; + maintainers = with maintainers; [ ryansydnor lopsided98 ]; + license = licenses.gpl2; + }; + }; + + waterfall-view = buildPythonPackage rec { + pname = "buildbot-waterfall-view"; + inherit (buildbot-pkg) version; + + src = fetchPypi { + inherit pname version; + hash = "sha256-935eeF2kpT68lK/UMg8MZQOYEj7D8FaT9iSs/lNahYA="; + }; + + buildInputs = [ buildbot-pkg ]; + + # No tests + doCheck = false; + + meta = with lib; { + homepage = "https://buildbot.net/"; + description = "Buildbot Waterfall View Plugin"; + maintainers = with maintainers; [ ryansydnor lopsided98 ]; + license = licenses.gpl2; + }; + }; + + grid-view = buildPythonPackage rec { + pname = "buildbot-grid-view"; + inherit (buildbot-pkg) version; + + src = fetchPypi { + inherit pname version; + hash = "sha256-3pHSiVoOZj2iCGfiz+tMWWMPHSBH5Ggp6e3+a8topsg="; + }; + + buildInputs = [ buildbot-pkg ]; + + # No tests + doCheck = false; + + meta = with lib; { + homepage = "https://buildbot.net/"; + description = "Buildbot Grid View Plugin"; + maintainers = with maintainers; [ lopsided98 ]; + license = licenses.gpl2; + }; + }; + + wsgi-dashboards = buildPythonPackage rec { + pname = "buildbot-wsgi-dashboards"; + inherit (buildbot-pkg) version; + + src = fetchPypi { + inherit pname version; + hash = "sha256-5Qr1FeYIJG/qaFaTB7ScFN9uca+joHKE6FlfKwhubfo="; + }; + + buildInputs = [ buildbot-pkg ]; + + # No tests + doCheck = false; + + meta = with lib; { + homepage = "https://buildbot.net/"; + description = "Buildbot WSGI dashboards Plugin"; + maintainers = with maintainers; [ lopsided98 ]; + license = licenses.gpl2; + }; + }; + + badges = buildPythonPackage rec { + pname = "buildbot-badges"; + inherit (buildbot-pkg) version; + + src = fetchPypi { + inherit pname version; + hash = "sha256-H0Dn+uTtFyZgyqbk3QQEc5t7CJovyzU+XuCoTe4Ajug="; + }; + + buildInputs = [ buildbot-pkg ]; + propagatedBuildInputs = [ cairosvg klein jinja2 ]; + + # No tests + doCheck = false; + + meta = with lib; { + homepage = "https://buildbot.net/"; + description = "Buildbot Badges Plugin"; + maintainers = with maintainers; [ julienmalka ]; + license = licenses.gpl2; + }; + }; + +} diff --git a/pkgs/development/tools/continuous-integration/buildbot/skip_test_linux_distro.patch b/pkgs/development/tools/continuous-integration/buildbot/skip_test_linux_distro.patch new file mode 100644 index 0000000000000..8fe5c7b56b4f2 --- /dev/null +++ b/pkgs/development/tools/continuous-integration/buildbot/skip_test_linux_distro.patch @@ -0,0 +1,11 @@ +diff -Nur buildbot-0.9.6/buildbot/test/unit/test_buildbot_net_usage_data.py buildbot-0.9.6.patched/buildbot/test/unit/test_buildbot_net_usage_data.py +--- buildbot-0.9.6/buildbot/test/unit/test_buildbot_net_usage_data.py 2017-04-19 16:57:02.000000000 +0200 ++++ buildbot-0.9.6.patched/buildbot/test/unit/test_buildbot_net_usage_data.py 2017-05-04 12:22:54.575762551 +0200 +@@ -147,6 +147,7 @@ + _sendBuildbotNetUsageData({'foo': 'bar'}) + + def test_linux_distro(self): ++ raise SkipTest("NixOS sandboxed builds hides /etc/os-release") + system = platform.system() + if system != "Linux": + raise SkipTest("test is only for linux") diff --git a/pkgs/development/tools/continuous-integration/buildbot/update.sh b/pkgs/development/tools/continuous-integration/buildbot/update.sh new file mode 100755 index 0000000000000..3406f05db3e80 --- /dev/null +++ b/pkgs/development/tools/continuous-integration/buildbot/update.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p nix-update +set -eu -o pipefail + +nix-update python3Packages.buildbot +nix-update --version=skip python3Packages.buildbot-worker +nix-update --version=skip python3Packages.buildbot-pkg +nix-update --version=skip python3Packages.buildbot-plugins.www +nix-update --version=skip python3Packages.buildbot-plugins.console-view +nix-update --version=skip python3Packages.buildbot-plugins.waterfall-view +nix-update --version=skip python3Packages.buildbot-plugins.grid-view +nix-update --version=skip python3Packages.buildbot-plugins.wsgi-dashboards diff --git a/pkgs/development/tools/continuous-integration/buildbot/worker.nix b/pkgs/development/tools/continuous-integration/buildbot/worker.nix new file mode 100644 index 0000000000000..3ea0feb293ec5 --- /dev/null +++ b/pkgs/development/tools/continuous-integration/buildbot/worker.nix @@ -0,0 +1,66 @@ +{ lib +, buildPythonPackage +, fetchPypi +, buildbot + +# patch +, coreutils + +# propagates +, autobahn +, future +, msgpack +, twisted + +# tests +, mock +, parameterized +, psutil +, setuptoolsTrial + +# passthru +, nixosTests +}: + +buildPythonPackage (rec { + pname = "buildbot-worker"; + inherit (buildbot) version; + + src = fetchPypi { + inherit pname version; + hash = "sha256-Lc+FNrfXLfeEnDqNBs9R96jtoFEOCG2vLRWGKip/+VM="; + }; + + postPatch = '' + substituteInPlace buildbot_worker/scripts/logwatcher.py \ + --replace /usr/bin/tail "${coreutils}/bin/tail" + ''; + + nativeBuildInputs = [ + setuptoolsTrial + ]; + + propagatedBuildInputs = [ + autobahn + future + msgpack + twisted + ]; + + nativeCheckInputs = [ + mock + parameterized + psutil + ]; + + passthru.tests = { + smoke-test = nixosTests.buildbot; + }; + + meta = with lib; { + homepage = "https://buildbot.net/"; + description = "Buildbot Worker Daemon"; + maintainers = with maintainers; [ ryansydnor lopsided98 ]; + license = licenses.gpl2; + }; +}) |