about summary refs log tree commit diff
path: root/pkgs/tools/misc/diffoscope/default.nix
blob: 6d97581af26f66f249d50eb9538a5f6acff46bfa (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
{ lib
, stdenv
, abootimg
, acl
, apksigcopier
, apksigner
, apktool
, binutils-unwrapped-all-targets
, bzip2
, cbfstool
, cdrkit
, colord
, colordiff
, coreutils
, cpio
, db
, diffutils
, docutils
, dtc
, e2fsprogs
, enableBloat ? true
, enjarify
, fetchurl
, file
, findutils
, fontforge-fonttools
, ffmpeg
, fpc
, gettext
, ghc
, ghostscriptX
, giflib
, gnumeric
, gnupg
, gnutar
, gzip
, hdf5
, help2man
, html2text
, imagemagick
, installShellFiles
, jdk
, libarchive
, libcaca
, libxmlb
, llvm
, lz4
, lzip
, mono
, ocaml
, odt2txt
, oggvideotools
, openssh
, openssl
, pdftk
, pgpdump
, poppler_utils
, procyon
, python3
, qemu
, R
, sng
, sqlite
, squashfsTools
, tcpdump
, ubootTools
, unzip
, wabt
, xmlbeans
, xxd
, xz
, zip
, zstd
  # updater only
, writeScript
}:

let
  python = python3.override {
    packageOverrides = final: prev: {
      # version 4 or newer would log the followng error but tests currently don't fail because radare2 is disabled
      # ValueError: argument TNULL is not a TLSH hex string
      tlsh = prev.tlsh.overridePythonAttrs ({ src, ... }: let
        version = "3.19.1";
      in {
        inherit version;
        src = src.override {
          rev = version;
          hash = "sha256-ZYEjT/yShfA4+zpbGOtaFOx1nSSOWPtMvskPhHv3c9U=";
        };
      });
    };
  };
in

# Note: when upgrading this package, please run the list-missing-tools.sh script as described below!
python.pkgs.buildPythonApplication rec {
  pname = "diffoscope";
  version = "265";

  src = fetchurl {
    url = "https://diffoscope.org/archive/diffoscope-${version}.tar.bz2";
    hash = "sha256-YE81R8lTOM3wmv/GIaIBqUq2O6UvnUaHjuXZ00yDU8U=";
  };

  outputs = [
    "out"
    "man"
  ];

  patches = [
    ./ignore_links.patch
    ./openssh-no-dsa.patch # https://salsa.debian.org/reproducible-builds/diffoscope/-/merge_requests/139
  ];

  postPatch = ''
    # Upstream doesn't provide a PKG-INFO file
    sed -i setup.py -e "/'rpm-python',/d"

    # When generating manpage, use the installed version
    substituteInPlace doc/Makefile --replace "../bin" "$out/bin"
  '';

  nativeBuildInputs = [
    docutils
    help2man
    installShellFiles
  ];

  # Most of the non-Python dependencies here are optional command-line tools for various file-format parsers.
  # To help figuring out what's missing from the list, run: ./pkgs/tools/misc/diffoscope/list-missing-tools.sh
  #
  # Still missing these tools:
  # Android-specific tools:
  # aapt2
  # dexdump
  # Darwin-specific tools:
  # lipo
  # otool
  # Other tools:
  # docx2txt <- makes tests broken:
  # > FAILED tests/comparators/test_docx.py::test_diff - IndexError: list index out of range
  # > FAILED tests/comparators/test_docx.py::test_compare_non_existing - AssertionError
  # radare2
  # > FAILED tests/comparators/test_elf_decompiler.py::test_ghidra_diff - IndexError: list index out of range
  # > FAILED tests/comparators/test_elf_decompiler.py::test_radare2_diff - AssertionError
  # > FAILED tests/comparators/test_macho_decompiler.py::test_ghidra_diff - assert 0 == 1
  # > FAILED tests/comparators/test_macho_decompiler.py::test_radare2_diff - AssertionError
  #
  # We filter automatically all packages for the host platform (some dependencies are not supported on Darwin, aarch64, etc.).
  pythonPath = lib.filter (lib.meta.availableOn stdenv.hostPlatform) ([
    acl
    binutils-unwrapped-all-targets
    bzip2
    cdrkit
    colordiff
    coreutils
    cpio
    db
    diffutils
    dtc
    e2fsprogs
    file
    findutils
    fontforge-fonttools
    gettext
    gnutar
    gzip
    html2text
    libarchive
    libxmlb
    lz4
    lzip
    openssl
    pgpdump
    sng
    sqlite
    squashfsTools
    unzip
    xxd
    xz
    zip
    zstd
  ]
  ++ (with python.pkgs; [
    argcomplete
    debian
    defusedxml
    jsbeautifier
    jsondiff
    libarchive-c
    progressbar33
    pypdf
    python-magic
    pyxattr
    rpm
    tlsh
  ])
  ++ lib.optionals enableBloat (
    [
      abootimg
      apksigcopier
      apksigner
      apktool
      cbfstool
      colord
      enjarify
      ffmpeg
      fpc
      ghc
      ghostscriptX
      giflib
      gnupg
      hdf5
      imagemagick
      jdk
      libcaca
      llvm
      mono
      ocaml
      odt2txt
      openssh
      pdftk
      poppler_utils
      procyon
      qemu
      R
      tcpdump
      ubootTools
      wabt
      xmlbeans
    ]
    ++ (with python.pkgs; [
      androguard
      binwalk
      guestfs
      h5py
      pdfminer-six
      r2pipe
      # docx2txt, nixpkgs packages another project named the same, which does not work
    ])
    # oggvideotools is broken on Darwin, please put it back when it will be fixed?
    ++ lib.optionals stdenv.isLinux [ oggvideotools ]
    # This doesn't work on aarch64-darwin
    ++ lib.optionals (stdenv.hostPlatform.system != "aarch64-darwin") [ gnumeric ]
  ));

  nativeCheckInputs = with python.pkgs; [
    pytestCheckHook
  ] ++ pythonPath;

  pytestFlagsArray = [
    # Always show more information when tests fail
    "-vv"
  ];

  postInstall = ''
    make -C doc
    installManPage doc/diffoscope.1
  '';

  disabledTests = [
    "test_sbin_added_to_path"
    "test_diff_meta"
    "test_diff_meta2"

    # Fails because it fails to determine llvm version
    "test_item3_deflate_llvm_bitcode"
  ] ++ lib.optionals stdenv.isDarwin [
    # Disable flaky tests on Darwin
    "test_non_unicode_filename"
    "test_listing"
    "test_symlink_root"

    # Appears to be a sandbox related issue
    "test_trim_stderr_in_command"
    # Seems to be a bug caused by having different versions of rdata than
    # expected. Will file upstream.
    "test_item_rdb"
    # Caused by getting an otool command instead of llvm-objdump. Could be Nix
    # setup, could be upstream bug. Will file upstream.
    "test_libmix_differences"
  ];

  disabledTestPaths = [
    # fails due to https://github.com/NixOS/nixpkgs/issues/256896
    # should be removed once that issue is resolved in coreboot or diffoscope
    "tests/comparators/test_cbfs.py"
  ]
  # Flaky tests on Darwin
  ++ lib.optionals stdenv.isDarwin [
    "tests/comparators/test_git.py"
    "tests/comparators/test_java.py"
    "tests/comparators/test_uimage.py"
    "tests/comparators/test_device.py"
    "tests/comparators/test_macho.py"
  ];

  passthru = {
    updateScript = writeScript "update-diffoscope" ''
      #!/usr/bin/env nix-shell
      #!nix-shell -i bash -p curl pcre common-updater-scripts

      set -eu -o pipefail

      # Expect the text in format of "Latest release: 198 (31 Dec 2021)"'.
      newVersion="$(curl -s https://diffoscope.org/ | pcregrep -o1 'Latest release: ([0-9]+)')"
      update-source-version ${pname} "$newVersion"
    '';
  };

  meta = with lib; {
    description = "Perform in-depth comparison of files, archives, and directories";
    longDescription = ''
      diffoscope will try to get to the bottom of what makes files or directories
      different. It will recursively unpack archives of many kinds and transform
      various binary formats into more human readable form to compare them. It can
      compare two tarballs, ISO images, or PDF just as easily. The differences can
      be shown in a text or HTML report.

      diffoscope is developed as part of the "reproducible builds" Debian
      project and was formerly known as "debbindiff".
    '';
    homepage = "https://diffoscope.org/";
    changelog = "https://diffoscope.org/news/diffoscope-${version}-released/";
    license = licenses.gpl3Plus;
    maintainers = with maintainers; [ dezgeg danielfullmer raitobezarius ];
    platforms = platforms.unix;
    mainProgram = "diffoscope";
  };
}