Merge pull request #155094 from fabaff/bump-grandalf

python3Packages.grandalf: 0.6 -> 0.7, dvc: 0.24.3 -> 2.9.3

12 files changed, 513 insertions, 60 deletions

diff --git a/pkgs/applications/version-management/dvc/default.nix b/pkgs/applications/version-management/dvc/default.nix
index ac1659cccf8bf..a886f6b332cc6 100644
--- a/pkgs/applications/version-management/dvc/default.nix
+++ b/pkgs/applications/version-management/dvc/default.nix
@@ -1,5 +1,5 @@
 { lib
-, python3Packages
+, python3
 , fetchFromGitHub
 , enableGoogle ? false
 , enableAWS ? false
@@ -7,59 +7,88 @@
 , enableSSH ? false
 }:
 
-with python3Packages;
-buildPythonApplication rec {
+python3.pkgs.buildPythonApplication rec {
   pname = "dvc";
-  version = "0.24.3";
+  version = "2.9.3";
+  format = "setuptools";
 
-  # PyPi only has wheel
   src = fetchFromGitHub {
     owner = "iterative";
-    repo = "dvc";
+    repo = pname;
     rev = version;
-    sha256 = "1wqq4i23hppilp20fx5a5nj93xwf3wwwr2f8aasvn6jkv2l22vpl";
+    hash = "sha256-nRlgo7Wjs7RgTUxoMYQh5YEsqiJtdWH2ex79rhXagAQ=";
   };
 
-  propagatedBuildInputs = [
-    ply
-    configparser
-    zc_lockfile
-    future
+  nativeBuildInputs = with python3.pkgs; [
+    setuptools-scm
+    setuptools-scm-git-archive
+  ];
+
+  propagatedBuildInputs = with python3.pkgs; [
+    appdirs
+    aiohttp-retry
     colorama
     configobj
-    networkx
-    pyyaml
-    GitPython
-    setuptools
+    configobj
+    dictdiffer
+    diskcache
+    distro
+    dpath
+    flatten-dict
+    flufl_lock
+    funcy
+    grandalf
     nanotime
-    pyasn1
-    schema
-    jsonpath_rw
+    networkx
+    pathspec
+    ply
+    psutil
+    pydot
+    pygtrie
+    pyparsing
+    python-benedict
     requests
-    grandalf
-    asciimatics
-    distro
-    appdirs
-  ]
-  ++ lib.optional enableGoogle google-cloud-storage
-  ++ lib.optional enableAWS boto3
-  ++ lib.optional enableAzure azure-storage-blob
-  ++ lib.optional enableSSH paramiko;
-
-  # tests require access to real cloud services
-  # nix build tests have to be isolated and run locally
-  doCheck = false;
+    rich
+    ruamel-yaml
+    scmrepo
+    shortuuid
+    shtab
+    tabulate
+    toml
+    tqdm
+    typing-extensions
+    voluptuous
+    zc_lockfile
+  ] ++ lib.optional enableGoogle [
+    google-cloud-storage
+  ] ++ lib.optional enableAWS [
+    boto3
+  ] ++ lib.optional enableAzure [
+    azure-storage-blob
+  ] ++ lib.optional enableSSH [
+    paramiko
+  ] ++ lib.optionals (pythonOlder "3.8") [
+    importlib-metadata
+  ] ++ lib.optionals (pythonOlder "3.9") [
+    importlib-resources
+  ];
 
   patches = [ ./dvc-daemon.patch ];
 
   postPatch = ''
-    substituteInPlace dvc/daemon.py --subst-var-by dvc "$out/bin/dcv"
+    substituteInPlace setup.cfg \
+      --replace "grandalf==0.6" "grandalf>=0.6"
+    substituteInPlace dvc/daemon.py \
+      --subst-var-by dvc "$out/bin/dcv"
   '';
 
+  # Tests require access to real cloud services
+  doCheck = false;
+
   meta = with lib; {
     description = "Version Control System for Machine Learning Projects";
-    license = licenses.asl20;
     homepage = "https://dvc.org";
-    maintainers = with maintainers; [ cmcdragonkai ];
+    license = licenses.asl20;
+    maintainers = with maintainers; [ cmcdragonkai fab ];
   };
 }
diff --git a/pkgs/applications/version-management/dvc/dvc-daemon.patch b/pkgs/applications/version-management/dvc/dvc-daemon.patch
index 5c2d363b17f10..2263ce118dd4f 100644
--- a/pkgs/applications/version-management/dvc/dvc-daemon.patch
+++ b/pkgs/applications/version-management/dvc/dvc-daemon.patch
@@ -1,21 +1,18 @@
 diff --git a/dvc/daemon.py b/dvc/daemon.py
-index 1d67a37..7ce6fde 100644
+index 9854a0e1..fefdd613 100644
 --- a/dvc/daemon.py
 +++ b/dvc/daemon.py
-@@ -67,14 +67,8 @@ def daemon(args):
-     Args:
-         args (list): list of arguments to append to `dvc daemon` command.
-     """
--    cmd = [sys.executable]
--    if not is_binary():
--        cmd += ['-m', 'dvc']
--    cmd += ['daemon', '-q'] + args
+@@ -103,11 +103,8 @@ def daemon(args):
+         logger.debug("skipping launching a new daemon.")
+         return
+ 
+-    cmd = ["daemon", "-q"] + args
 -
 -    env = fix_env()
 -    file_path = os.path.abspath(inspect.stack()[0][1])
--    env['PYTHONPATH'] = os.path.dirname(os.path.dirname(file_path))
+-    env["PYTHONPATH"] = os.path.dirname(os.path.dirname(file_path))
 +    cmd = [ "@dvc@" , "daemon", "-q"] + args
 +    env = None
+     env[DVC_DAEMON] = "1"
  
-     logger.debug("Trying to spawn '{}' with env '{}'".format(cmd, env))
- 
+     _spawn(cmd, env)
diff --git a/pkgs/development/python-modules/aiohttp-retry/default.nix b/pkgs/development/python-modules/aiohttp-retry/default.nix
new file mode 100644
index 0000000000000..3cd0fe0984267
--- /dev/null
+++ b/pkgs/development/python-modules/aiohttp-retry/default.nix
@@ -0,0 +1,43 @@
+{ lib
+, aiohttp
+, buildPythonPackage
+, fetchFromGitHub
+, pytestCheckHook
+, pytest-aiohttp
+, pythonOlder
+}:
+
+buildPythonPackage rec {
+  pname = "aiohttp-retry";
+  version = "2.5.6";
+  format = "setuptools";
+
+  disabled = pythonOlder "3.7";
+
+  src = fetchFromGitHub {
+    owner = "inyutin";
+    repo = "aiohttp_retry";
+    rev = "v${version}";
+    hash = "sha256-jyt4YPn3gSgR1YfHYLs+5VCsjAk9Ij+2m5Kzy51CnLk=";
+  };
+
+  propagatedBuildInputs = [
+    aiohttp
+  ];
+
+  checkInputs = [
+    pytest-aiohttp
+    pytestCheckHook
+  ];
+
+  pythonImportsCheck = [
+    "aiohttp_retry"
+  ];
+
+  meta = with lib; {
+    description = "Retry client for aiohttp";
+    homepage = "https://github.com/inyutin/aiohttp_retry";
+    license = licenses.mit;
+    maintainers = with maintainers; [ fab ];
+  };
+}
diff --git a/pkgs/development/python-modules/dictdiffer/default.nix b/pkgs/development/python-modules/dictdiffer/default.nix
new file mode 100644
index 0000000000000..5e043e55f853e
--- /dev/null
+++ b/pkgs/development/python-modules/dictdiffer/default.nix
@@ -0,0 +1,50 @@
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+, pytestCheckHook
+, pythonOlder
+, setuptools-scm
+}:
+
+buildPythonPackage rec {
+  pname = "dictdiffer";
+  version = "0.9.0";
+  format = "setuptools";
+
+  disabled = pythonOlder "3.7";
+
+  src = fetchFromGitHub {
+    owner = "inveniosoftware";
+    repo = pname;
+    rev = "v${version}";
+    hash = "sha256-lQyPs3lQWtsvNPuvvwJUTDzrFaOX5uwGuRHe3yWUheU=";
+  };
+
+  SETUPTOOLS_SCM_PRETEND_VERSION = version;
+
+  nativeBuildInputs = [
+    setuptools-scm
+  ];
+
+  checkInputs = [
+    pytestCheckHook
+  ];
+
+  postPatch = ''
+    substituteInPlace setup.py \
+      --replace "'pytest-runner>=2.7'," ""
+    substituteInPlace pytest.ini \
+      --replace ' --isort --pydocstyle --pycodestyle --doctest-glob="*.rst" --doctest-modules --cov=dictdiffer --cov-report=term-missing' ""
+  '';
+
+  pythonImportsCheck = [
+    "dictdiffer"
+  ];
+
+  meta = with lib; {
+    description = "Module to diff and patch dictionaries";
+    homepage = "https://github.com/inveniosoftware/dictdiffer";
+    license = licenses.mit;
+    maintainers = with maintainers; [ fab ];
+  };
+}
diff --git a/pkgs/development/python-modules/flatten-dict/default.nix b/pkgs/development/python-modules/flatten-dict/default.nix
new file mode 100644
index 0000000000000..e7bda9378b192
--- /dev/null
+++ b/pkgs/development/python-modules/flatten-dict/default.nix
@@ -0,0 +1,46 @@
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+, poetry-core
+, pytestCheckHook
+, pythonOlder
+, six
+}:
+
+buildPythonPackage rec {
+  pname = "flatten-dict";
+  version = "0.4.2";
+  format = "pyproject";
+
+  disabled = pythonOlder "3.7";
+
+  src = fetchFromGitHub {
+    owner = "ianlini";
+    repo = pname;
+    rev = version;
+    hash = "sha256-uHenKoD4eLm9sMREVuV0BB/oUgh4NMiuj+IWd0hlxNQ=";
+  };
+
+  nativeBuildInputs = [
+    poetry-core
+  ];
+
+  propagatedBuildInputs = [
+    six
+  ];
+
+  checkInputs = [
+    pytestCheckHook
+  ];
+
+  pythonImportsCheck = [
+    "flatten_dict"
+  ];
+
+  meta = with lib; {
+    description = "Module for flattening and unflattening dict-like objects";
+    homepage = "https://github.com/ianlini/flatten-dict";
+    license = licenses.mit;
+    maintainers = with maintainers; [ fab ];
+  };
+}
diff --git a/pkgs/development/python-modules/grandalf/default.nix b/pkgs/development/python-modules/grandalf/default.nix
index 6056d215d2276..c4e7eb4d74462 100644
--- a/pkgs/development/python-modules/grandalf/default.nix
+++ b/pkgs/development/python-modules/grandalf/default.nix
@@ -3,20 +3,22 @@
 , fetchFromGitHub
 , pyparsing
 , future
-, pytest
-, pytest-runner
+, pytestCheckHook
+, pythonOlder
 }:
 
 buildPythonPackage rec {
   pname = "grandalf";
-  version = "0.6";
+  version = "0.7";
+  format = "setuptools";
+
+  disabled = pythonOlder "3.7";
 
-  # fetch from github to acquire tests
   src = fetchFromGitHub {
     owner = "bdcht";
-    repo = "grandalf";
+    repo = pname;
     rev = "v${version}";
-    sha256 = "1f1l288sqna0bca7dwwvyw7wzg9b2613g6vc0g0vfngm7k75b2jg";
+    hash = "sha256-j2SvpQvDMfwoj2PAQSxzEIyIzzJ61Eb9wgetKyni6A4=";
   };
 
   propagatedBuildInputs = [
@@ -24,18 +26,22 @@ buildPythonPackage rec {
     future
   ];
 
-  checkInputs = [ pytest pytest-runner ];
+  checkInputs = [
+    pytestCheckHook
+  ];
 
-  patches = [ ./no-setup-requires-pytestrunner.patch ];
+  patches = [
+    ./no-setup-requires-pytestrunner.patch
+  ];
 
-  checkPhase = ''
-    pytest tests
-  '';
+  pythonImportsCheck = [
+    "grandalf"
+  ];
 
   meta = with lib; {
-    description = "A python package made for experimentations with graphs and drawing algorithms";
+    description = "Module for experimentations with graphs and drawing algorithms";
     homepage = "https://github.com/bdcht/grandalf";
-    license = licenses.gpl2;
+    license = licenses.gpl2Only;
     maintainers = with maintainers; [ cmcdragonkai ];
   };
 }
diff --git a/pkgs/development/python-modules/mailchecker/default.nix b/pkgs/development/python-modules/mailchecker/default.nix
new file mode 100644
index 0000000000000..ce47d6b3a2d5e
--- /dev/null
+++ b/pkgs/development/python-modules/mailchecker/default.nix
@@ -0,0 +1,32 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, pythonOlder
+}:
+
+buildPythonPackage rec {
+  pname = "mailchecker";
+  version = "4.1.8";
+  format = "setuptools";
+
+  disabled = pythonOlder "3.7";
+
+  src = fetchPypi {
+    inherit pname version;
+    hash = "sha256-RLZunjRX7lljKQl+sJ/py8bTR/YZsDewTJao9IBuLbE=";
+  };
+
+  # Module has no tests
+  doCheck = false;
+
+  pythonImportsCheck = [
+    "MailChecker"
+  ];
+
+  meta = with lib; {
+    description = "Module for temporary (disposable/throwaway) email detection";
+    homepage = "https://github.com/FGRibreau/mailchecker";
+    license = licenses.mit;
+    maintainers = with maintainers; [ fab ];
+  };
+}
diff --git a/pkgs/development/python-modules/python-benedict/default.nix b/pkgs/development/python-modules/python-benedict/default.nix
new file mode 100644
index 0000000000000..9cf5ffc63abf9
--- /dev/null
+++ b/pkgs/development/python-modules/python-benedict/default.nix
@@ -0,0 +1,75 @@
+{ lib
+, aiohttp
+, buildPythonPackage
+, fetchFromGitHub
+, pytestCheckHook
+, pythonOlder
+
+, mailchecker
+, phonenumbers
+, python-dateutil
+, python-fsutil
+, python-slugify
+, pyyaml
+, ftfy
+, requests
+, six
+, toml
+, xmltodict
+}:
+
+buildPythonPackage rec {
+  pname = "python-benedict";
+  version = "0.24.3";
+  format = "setuptools";
+
+  disabled = pythonOlder "3.7";
+
+  src = fetchFromGitHub {
+    owner = "fabiocaccamo";
+    repo = pname;
+    rev = version;
+    hash = "sha256-06n8MNoGQRSrBK2XeEBBoQ2NIXWf0qXPVBeP9ERMEj0=";
+  };
+
+  propagatedBuildInputs = [
+    mailchecker
+    phonenumbers
+    python-dateutil
+    python-fsutil
+    python-slugify
+    pyyaml
+    ftfy
+    requests
+    six
+    toml
+    xmltodict
+  ];
+
+  checkInputs = [
+    pytestCheckHook
+  ];
+
+  disabledTests = [
+    # Tests require network access
+    "test_from_base64_with_valid_url_valid_content"
+    "test_from_json_with_valid_url_valid_content"
+    "test_from_pickle_with_valid_url_valid_content"
+    "test_from_plist_with_valid_url_valid_content"
+    "test_from_query_string_with_valid_url_valid_content"
+    "test_from_toml_with_valid_url_valid_content"
+    "test_from_xml_with_valid_url_valid_content"
+    "test_from_yaml_with_valid_url_valid_content"
+  ];
+
+  pythonImportsCheck = [
+    "benedict"
+  ];
+
+  meta = with lib; {
+    description = "Module with keylist/keypath support";
+    homepage = "https://github.com/fabiocaccamo/python-benedict";
+    license = licenses.mit;
+    maintainers = with maintainers; [ fab ];
+  };
+}
diff --git a/pkgs/development/python-modules/python-fsutil/default.nix b/pkgs/development/python-modules/python-fsutil/default.nix
new file mode 100644
index 0000000000000..17dbd579e8411
--- /dev/null
+++ b/pkgs/development/python-modules/python-fsutil/default.nix
@@ -0,0 +1,51 @@
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+, pytestCheckHook
+, pythonOlder
+, requests
+}:
+
+buildPythonPackage rec {
+  pname = "python-fsutil";
+  version = "0.5.0";
+  format = "setuptools";
+
+  disabled = pythonOlder "3.7";
+
+  src = fetchFromGitHub {
+    owner = "fabiocaccamo";
+    repo = pname;
+    rev = version;
+    hash = "sha256-zWthL7iwdVzdihX2YA4G//B18iwe1gRT0GM2KNP01kQ=";
+  };
+
+  propagatedBuildInputs = [
+    requests
+  ];
+
+  checkInputs = [
+    pytestCheckHook
+  ];
+
+  pytestFlagsArray = [
+    "tests/test.py"
+  ];
+
+  disabledTests = [
+    # Tests require network access
+    "test_download_file"
+    "test_read_file_from_url"
+  ];
+
+  pythonImportsCheck = [
+    "fsutil"
+  ];
+
+  meta = with lib; {
+    description = "Module with file-system utilities";
+    homepage = "https://github.com/fabiocaccamo/python-fsutil";
+    license = licenses.mit;
+    maintainers = with maintainers; [ fab ];
+  };
+}
diff --git a/pkgs/development/python-modules/scmrepo/default.nix b/pkgs/development/python-modules/scmrepo/default.nix
new file mode 100644
index 0000000000000..f4a024a46e99e
--- /dev/null
+++ b/pkgs/development/python-modules/scmrepo/default.nix
@@ -0,0 +1,53 @@
+{ lib
+, asyncssh
+, buildPythonPackage
+, dulwich
+, fetchFromGitHub
+, fsspec
+, funcy
+, GitPython
+, pathspec
+, pygit2
+, pygtrie
+, pythonOlder
+}:
+
+buildPythonPackage rec {
+  pname = "scmrepo";
+  version = "0.0.7";
+  format = "pyproject";
+
+  disabled = pythonOlder "3.7";
+
+  src = fetchFromGitHub {
+    owner = "iterative";
+    repo = pname;
+    rev = version;
+    hash = "sha256-tZsogqcfAqpSo9yOz4z0mgY9SVU1epPmcBuyLJsHLfY=";
+  };
+
+  propagatedBuildInputs = [
+    asyncssh
+    dulwich
+    fsspec
+    funcy
+    GitPython
+    pathspec
+    pygit2
+    pygtrie
+  ];
+
+  # Requires a running Docker instance
+  doCheck = false;
+
+  pythonImportsCheck = [
+    "scmrepo"
+  ];
+
+  meta = with lib; {
+    description = "SCM wrapper and fsspec filesystem";
+    homepage = "https://github.com/iterative/scmrepo";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ fab ];
+  };
+}
diff --git a/pkgs/development/python-modules/shtab/default.nix b/pkgs/development/python-modules/shtab/default.nix
new file mode 100644
index 0000000000000..efac8148a72ac
--- /dev/null
+++ b/pkgs/development/python-modules/shtab/default.nix
@@ -0,0 +1,55 @@
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+, pytest-timeout
+, pytestCheckHook
+, pythonOlder
+, setuptools-scm
+}:
+
+buildPythonPackage rec {
+  pname = "shtab";
+  version = "1.5.3";
+  format = "setuptools";
+
+  disabled = pythonOlder "3.7";
+
+  src = fetchFromGitHub {
+    owner = "iterative";
+    repo = pname;
+    rev = "v${version}";
+    hash = "sha256-Wuc4m3VdOGEcevYXUpbL4gTvyW9t13pj57zPYdqx0UY=";
+  };
+
+  SETUPTOOLS_SCM_PRETEND_VERSION = version;
+
+  nativeBuildInputs = [
+    setuptools-scm
+  ];
+
+  checkInputs = [
+    pytest-timeout
+    pytestCheckHook
+  ];
+
+  postPatch = ''
+    substituteInPlace setup.cfg \
+      --replace " --cov=shtab --cov-report=term-missing --cov-report=xml" ""
+  '';
+
+  disabledTests = [
+    # bash tests are failing
+    "bash"
+  ];
+
+  pythonImportsCheck = [
+    "shtab"
+  ];
+
+  meta = with lib; {
+    description = "Module for shell tab completion of Python CLI applications";
+    homepage = "https://docs.iterative.ai/shtab/";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ fab ];
+  };
+}
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index 7bfe838ab0824..0a37eb35520f5 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -310,6 +310,8 @@ in {
 
   aiohttp-remotes = callPackage ../development/python-modules/aiohttp-remotes { };
 
+  aiohttp-retry = callPackage ../development/python-modules/aiohttp-retry { };
+
   aiohttp-socks = callPackage ../development/python-modules/aiohttp-socks { };
 
   aiohttp-swagger = callPackage ../development/python-modules/aiohttp-swagger { };
@@ -2118,6 +2120,8 @@ in {
 
   dict2xml = callPackage ../development/python-modules/dict2xml { };
 
+  dictdiffer = callPackage ../development/python-modules/dictdiffer { };
+
   dictionaries = callPackage ../development/python-modules/dictionaries { };
 
   dictpath = callPackage ../development/python-modules/dictpath { };
@@ -2948,6 +2952,8 @@ in {
     inherit (pkgs) flatbuffers;
   };
 
+  flatten-dict = callPackage ../development/python-modules/flatten-dict { };
+
   flax = callPackage ../development/python-modules/flax { };
 
   flexmock = callPackage ../development/python-modules/flexmock { };
@@ -4762,6 +4768,8 @@ in {
 
   mailcap-fix = callPackage ../development/python-modules/mailcap-fix { };
 
+  mailchecker = callPackage ../development/python-modules/mailchecker { };
+
   mailchimp = callPackage ../development/python-modules/mailchimp { };
 
   mailman = callPackage ../servers/mail/mailman { };
@@ -7784,6 +7792,8 @@ in {
 
   python-baseconv = callPackage ../development/python-modules/python-baseconv { };
 
+  python-benedict = callPackage ../development/python-modules/python-benedict { };
+
   python-bidi = callPackage ../development/python-modules/python-bidi { };
 
   python-binance = callPackage ../development/python-modules/python-binance { };
@@ -7818,6 +7828,8 @@ in {
 
   python-editor = callPackage ../development/python-modules/python-editor { };
 
+  python-fsutil = callPackage ../development/python-modules/python-fsutil { };
+
   pythonefl = callPackage ../development/python-modules/python-efl { };
 
   pythonegardia = callPackage ../development/python-modules/pythonegardia { };
@@ -8749,6 +8761,8 @@ in {
 
   scipy = callPackage ../development/python-modules/scipy { };
 
+  scmrepo = callPackage ../development/python-modules/scmrepo { };
+
   scour = callPackage ../development/python-modules/scour { };
 
   scp = callPackage ../development/python-modules/scp { };
@@ -8896,6 +8910,8 @@ in {
 
   showit = callPackage ../development/python-modules/showit { };
 
+  shtab = callPackage ../development/python-modules/shtab { };
+
   shutilwhich = callPackage ../development/python-modules/shutilwhich { };
 
   sievelib = callPackage ../development/python-modules/sievelib { };