1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
|
{ lib
, stdenv
, buildPythonPackage
, braceexpand
, imageio
, lmdb
, msgpack
, numpy
, pytestCheckHook
, pyyaml
, setuptools
, torch
, torchvision
, wheel
, fetchFromGitHub
}:
buildPythonPackage rec {
pname = "webdataset";
version = "0.2.90";
pyproject = true;
src = fetchFromGitHub {
owner = "webdataset";
repo = "webdataset";
rev = "refs/tags/${version}";
hash = "sha256-selj7XD7NS831lbPnx/4o46bNpsxuFdSEIIb4S2b7S0=";
};
nativeBuildInputs = [
setuptools
wheel
];
propagatedBuildInputs = [
braceexpand
numpy
pyyaml
];
nativeCheckInputs = [
pytestCheckHook
imageio
torch
torchvision
msgpack
lmdb
];
pythonImportsCheck = [ "webdataset" ];
disabledTests = [
# requires network
"test_batched"
"test_cache_dir"
"test_concurrent_download_and_open"
"test_dataloader"
"test_decode_handlers"
"test_decoder"
"test_download"
"test_handlers"
"test_pipe"
"test_remote_file"
"test_shard_syntax"
"test_torchvision"
"test_unbatched"
"test_yaml3"
] ++ lib.optionals stdenv.isDarwin [
# pickling error
"test_background_download"
] ++ lib.optionals (stdenv.isx86_64 && stdenv.isDarwin) [
"test_concurrent_access"
# fails to patch 'init_process_group' from torch.distributed
"TestDistributedChunkedSampler"
] ++ lib.optionals (stdenv.isAarch64 && stdenv.isLinux) [
# segfaults on aarch64-linux
"test_webloader"
"test_webloader2"
"test_webloader_repeat"
"test_webloader_unbatched"
];
meta = with lib; {
description = "A high-performance Python-based I/O system for large (and small) deep learning problems, with strong support for PyTorch";
mainProgram = "widsindex";
homepage = "https://github.com/webdataset/webdataset";
changelog = "https://github.com/webdataset/webdataset/releases/tag/${version}";
license = licenses.bsd3;
maintainers = with maintainers; [ iynaix ];
};
}
|