diff options
Diffstat (limited to 'pkgs/servers/invidious')
-rw-r--r-- | pkgs/servers/invidious/default.nix | 53 | ||||
-rw-r--r-- | pkgs/servers/invidious/lsquic.nix | 20 | ||||
-rw-r--r-- | pkgs/servers/invidious/shards.nix | 12 | ||||
-rwxr-xr-x | pkgs/servers/invidious/update.sh | 96 | ||||
-rw-r--r-- | pkgs/servers/invidious/use-etc-ssl-certs.patch | 13 | ||||
-rw-r--r-- | pkgs/servers/invidious/versions.json | 18 | ||||
-rw-r--r-- | pkgs/servers/invidious/videojs.nix | 18 | ||||
-rw-r--r-- | pkgs/servers/invidious/videojs.sh | 9 |
8 files changed, 213 insertions, 26 deletions
diff --git a/pkgs/servers/invidious/default.nix b/pkgs/servers/invidious/default.nix index 7e939f51bc952..adcd694bad946 100644 --- a/pkgs/servers/invidious/default.nix +++ b/pkgs/servers/invidious/default.nix @@ -1,24 +1,27 @@ -{ lib, crystal, fetchFromGitHub, librsvg, pkg-config, libxml2, openssl, sqlite, lsquic, nixosTests }: +{ lib, stdenv, crystal, fetchFromGitHub, librsvg, pkg-config, libxml2, openssl, shards, sqlite, lsquic, videojs, nixosTests }: let - # When updating, always update the following: - # * the git revision - # * the version attribute - # * the source hash (sha256) - # If the shards.lock file changed, also the following: - # * shards.nix (by running `crystal2nix` in invidious’ source tree) - # * If the lsquic.cr dependency changed: lsquic in lsquic.nix (version, sha256) - # * If the lsquic version changed: boringssl' in lsquic.nix (version, sha256) - rev = "21879da80d2dfa97e789a13b90e82e466c4854e3"; + # All versions, revisions, and checksums are stored in ./versions.json. + # The update process is the following: + # * pick the latest commit + # * update .invidious.rev, .invidious.version, and .invidious.sha256 + # * prefetch the videojs dependencies with scripts/fetch-player-dependencies.cr + # and update .videojs.sha256 (they are normally fetched during build + # but nix's sandboxing does not allow that) + # * if shard.lock changed + # * recreate shards.nix by running crystal2nix + # * update lsquic and boringssl if necessarry, lsquic.cr depends on + # the same version of lsquic and lsquic requires the boringssl + # commit mentioned in its README + versions = builtins.fromJSON (builtins.readFile ./versions.json); in crystal.buildCrystalPackage rec { pname = "invidious"; - version = "unstable-2021-11-08"; + inherit (versions.invidious) version; src = fetchFromGitHub { owner = "iv-org"; repo = pname; - inherit rev; - sha256 = "0jvnwjdh2l0hxfvzim00r3zbs528bb93y1nk0bjrbbrcfv5cn5ss"; + inherit (versions.invidious) rev sha256; }; postPatch = @@ -33,13 +36,15 @@ crystal.buildCrystalPackage rec { assetCommitTemplate = ''{{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit -- assets`.strip}" }}''; in '' + for d in ${videojs}/*; do ln -s "$d" assets/videojs; done + # Use the version metadata from the derivation instead of using git at # build-time substituteInPlace src/invidious.cr \ --replace ${lib.escapeShellArg branchTemplate} '"master"' \ - --replace ${lib.escapeShellArg commitTemplate} '"${lib.substring 0 7 rev}"' \ + --replace ${lib.escapeShellArg commitTemplate} '"${lib.substring 0 7 versions.invidious.rev}"' \ --replace ${lib.escapeShellArg versionTemplate} '"${lib.replaceChars ["-"] ["."] (lib.substring 9 10 version)}"' \ - --replace ${lib.escapeShellArg assetCommitTemplate} '"${lib.substring 0 7 rev}"' + --replace ${lib.escapeShellArg assetCommitTemplate} '"${lib.substring 0 7 versions.invidious.rev}"' # Patch the assets and locales paths to be absolute substituteInPlace src/invidious.cr \ @@ -48,19 +53,22 @@ crystal.buildCrystalPackage rec { --replace 'File.read("locales/' 'File.read("${placeholder "out"}/share/invidious/locales/' # Reference sql initialisation/migration scripts by absolute path - substituteInPlace src/invidious/helpers/helpers.cr \ + substituteInPlace src/invidious/database/base.cr \ --replace 'config/sql' '${placeholder "out"}/share/invidious/config/sql' - substituteInPlace src/invidious/users.cr \ + substituteInPlace src/invidious/user/captcha.cr \ --replace 'Process.run(%(rsvg-convert' 'Process.run(%(${lib.getBin librsvg}/bin/rsvg-convert' ''; - nativeBuildInputs = [ pkg-config ]; + nativeBuildInputs = [ pkg-config shards ]; buildInputs = [ libxml2 openssl sqlite ]; format = "crystal"; shardsFile = ./shards.nix; - crystalBinaries.invidious.src = "src/invidious.cr"; + crystalBinaries.invidious = { + src = "src/invidious.cr"; + options = [ "--release" "--progress" "--verbose" "--no-debug" "-Dskip_videojs_download" ]; + }; postConfigure = '' # lib includes nix store paths which can’t be patched, so the links have to @@ -88,12 +96,17 @@ crystal.buildCrystalPackage rec { INVIDIOUS_CONFIG="database_url: sqlite3:///dev/null" $out/bin/invidious --help ''; - passthru.tests = { inherit (nixosTests) invidious; }; + passthru = { + inherit lsquic; + tests = { inherit (nixosTests) invidious; }; + updateScript = ./update.sh; + }; meta = with lib; { description = "An open source alternative front-end to YouTube"; homepage = "https://invidious.io/"; license = licenses.agpl3; maintainers = with maintainers; [ infinisil sbruder ]; + broken = stdenv.isDarwin && stdenv.isAarch64; }; } diff --git a/pkgs/servers/invidious/lsquic.nix b/pkgs/servers/invidious/lsquic.nix index acc9b97ae5df7..292d37ce6cd58 100644 --- a/pkgs/servers/invidious/lsquic.nix +++ b/pkgs/servers/invidious/lsquic.nix @@ -1,24 +1,30 @@ { lib, boringssl, stdenv, fetchgit, fetchFromGitHub, cmake, zlib, perl, libevent }: let + versions = builtins.fromJSON (builtins.readFile ./versions.json); + # lsquic requires a specific boringssl version (noted in its README) - boringssl' = boringssl.overrideAttrs (old: rec { - version = "251b5169fd44345f455438312ec4e18ae07fd58c"; + boringssl' = boringssl.overrideAttrs (old: { + version = versions.boringssl.rev; src = fetchgit { url = "https://boringssl.googlesource.com/boringssl"; - rev = version; - sha256 = "sha256-EU6T9yQCdOLx98Io8o01rEsgxDFF/Xoy42LgPopD2/A="; + inherit (versions.boringssl) rev sha256; }; + + patches = [ + # Use /etc/ssl/certs/ca-certificates.crt instead of /etc/ssl/cert.pem + ./use-etc-ssl-certs.patch + ]; }); in stdenv.mkDerivation rec { pname = "lsquic"; - version = "2.18.1"; + version = versions.lsquic.version; src = fetchFromGitHub { owner = "litespeedtech"; repo = pname; rev = "v${version}"; - sha256 = "sha256-hG8cUvhbCNeMOsKkaJlgGpzUrIx47E/WhmPIdI5F3qM="; + inherit (versions.lsquic) sha256; fetchSubmodules = true; }; @@ -49,6 +55,8 @@ stdenv.mkDerivation rec { runHook postInstall ''; + passthru.boringssl = boringssl'; + meta = with lib; { description = "A library for QUIC and HTTP/3 (version for Invidious)"; homepage = "https://github.com/litespeedtech/lsquic"; diff --git a/pkgs/servers/invidious/shards.nix b/pkgs/servers/invidious/shards.nix index 778b34496a628..582a4083e7db3 100644 --- a/pkgs/servers/invidious/shards.nix +++ b/pkgs/servers/invidious/shards.nix @@ -59,10 +59,22 @@ rev = "v0.4.1"; sha256 = "1l08cydkdidq9yyil1wl240hvk41iycv04jrg6nx5mkvzw4z1bzg"; }; + spectator = { + owner = "icy-arctic-fox"; + repo = "spectator"; + rev = "v0.10.4"; + sha256 = "0rcxq2nbslvwrd8m9ajw6dzaw3hagxmkdy9s8p34cgnr4c9dijdq"; + }; sqlite3 = { owner = "crystal-lang"; repo = "crystal-sqlite3"; rev = "v0.18.0"; sha256 = "03nnvpchhq9f9ywsm3pk2rrj4a3figw7xs96zdziwgr5znkz6x93"; }; + ameba = { + owner = "crystal-ameba"; + repo = "ameba"; + rev = "v0.14.3"; + sha256 = "1cfr95xi6hsyxw1wlrh571hc775xhwmssk3k14i8b7dgbwfmm5x1"; + }; } diff --git a/pkgs/servers/invidious/update.sh b/pkgs/servers/invidious/update.sh new file mode 100755 index 0000000000000..580d6136388a9 --- /dev/null +++ b/pkgs/servers/invidious/update.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p curl crystal crystal2nix jq git moreutils nix nix-prefetch pkg-config +git_url='https://github.com/iv-org/invidious.git' +git_branch='master' +git_dir='/var/tmp/invidious.git' +pkg='invidious' + +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")" + +info() { + if [ -t 2 ]; then + set -- '\033[32m%s\033[39m\n' "$@" + else + set -- '%s\n' "$@" + fi + printf "$@" >&2 +} + +json_get() { + jq -r "$1" < 'versions.json' +} + +json_set() { + jq --arg x "$2" "$1 = \$x" < 'versions.json' | sponge 'versions.json' +} + +old_rev=$(json_get '.invidious.rev') +old_version=$(json_get '.invidious.version') +today=$(LANG=C date -u +'%Y-%m-%d') + +info "fetching $git_url..." +if [ ! -d "$git_dir" ]; then + git init --initial-branch="$git_branch" "$git_dir" + git -C "$git_dir" remote add origin "$git_url" +fi +git -C "$git_dir" fetch origin "$git_branch" + +# use latest commit before today, we should not call the version *today* +# because there might still be commits coming +# use the day of the latest commit we picked as version +new_rev=$(git -C "$git_dir" log -n 1 --format='format:%H' --before="${today}T00:00:00Z" "origin/$git_branch") +new_version="unstable-$(git -C "$git_dir" log -n 1 --format='format:%cs' "$new_rev")" +info "latest commit before $today: $new_rev" + +if [ "$new_rev" = "$old_rev" ]; then + info "$pkg is up-to-date." + exit +fi + +json_set '.invidious.version' "$new_version" +json_set '.invidious.rev' "$new_rev" +new_sha256=$(nix-prefetch -I 'nixpkgs=../../..' "$pkg") +json_set '.invidious.sha256' "$new_sha256" +commit_msg="$pkg: $old_version -> $new_version" + +# fetch video.js dependencies +info "Running scripts/fetch-player-dependencies.cr..." +git -C "$git_dir" reset --hard "$new_rev" +(cd "$git_dir" && crystal run scripts/fetch-player-dependencies.cr -- --minified) +rm -f "$git_dir/assets/videojs/.gitignore" +videojs_new_sha256=$(nix hash-path --type sha256 --base32 "$git_dir/assets/videojs") +json_set '.videojs.sha256' "$videojs_new_sha256" + +if git -C "$git_dir" diff-tree --quiet "${old_rev}..${new_rev}" -- 'shard.lock'; then + info "shard.lock did not change since $old_rev." +else + info "Updating shards.nix..." + crystal2nix -- "$git_dir/shard.lock" # argv's index seems broken + + lsquic_old_version=$(json_get '.lsquic.version') + # lsquic.cr's version tracks lsquic's, so lsquic must be updated to the + # version in the shards file + lsquic_new_version=$(nix eval --raw -f 'shards.nix' lsquic.rev \ + | sed -e 's/^v//' -e 's/-[0-9]*$//') + if [ "$lsquic_old_version" != "$lsquic_new_version" ]; then + info "Updating lsquic to $lsquic_new_version..." + json_set '.lsquic.version' "$lsquic_new_version" + lsquic_new_sha256=$(nix-prefetch -I 'nixpkgs=../../..' "${pkg}.lsquic") + json_set '.lsquic.sha256' "$lsquic_new_sha256" + + info "Updating boringssl..." + # lsquic specifies the boringssl commit it requires in its README + boringssl_new_rev=$(curl -LSsf "https://github.com/litespeedtech/lsquic/raw/v${lsquic_new_version}/README.md" \ + | grep -Pom1 '(?<=^git checkout ).*') + json_set '.boringssl.rev' "$boringssl_new_rev" + boringssl_new_sha256=$(nix-prefetch -I 'nixpkgs=../../..' "${pkg}.lsquic.boringssl") + json_set '.boringssl.sha256' "$boringssl_new_sha256" + commit_msg="$commit_msg + +lsquic: $lsquic_old_version -> $lsquic_new_version" + fi +fi + +git commit --verbose --message "$commit_msg" -- versions.json shards.nix diff --git a/pkgs/servers/invidious/use-etc-ssl-certs.patch b/pkgs/servers/invidious/use-etc-ssl-certs.patch new file mode 100644 index 0000000000000..b60b0b1f1a122 --- /dev/null +++ b/pkgs/servers/invidious/use-etc-ssl-certs.patch @@ -0,0 +1,13 @@ +diff --git a/crypto/x509/x509_def.c b/crypto/x509/x509_def.c +index d2bc3e5c1..329580075 100644 +--- a/crypto/x509/x509_def.c ++++ b/crypto/x509/x509_def.c +@@ -67,7 +67,7 @@ + + #define X509_CERT_AREA OPENSSLDIR + #define X509_CERT_DIR OPENSSLDIR "/certs" +-#define X509_CERT_FILE OPENSSLDIR "/cert.pem" ++#define X509_CERT_FILE "/etc/ssl/certs/ca-certificates.crt" + #define X509_PRIVATE_DIR OPENSSLDIR "/private" + #define X509_CERT_DIR_EVP "SSL_CERT_DIR" + #define X509_CERT_FILE_EVP "SSL_CERT_FILE" diff --git a/pkgs/servers/invidious/versions.json b/pkgs/servers/invidious/versions.json new file mode 100644 index 0000000000000..cec068a09bf1f --- /dev/null +++ b/pkgs/servers/invidious/versions.json @@ -0,0 +1,18 @@ +{ + "boringssl": { + "rev": "251b5169fd44345f455438312ec4e18ae07fd58c", + "sha256": "sha256-EU6T9yQCdOLx98Io8o01rEsgxDFF/Xoy42LgPopD2/A=" + }, + "invidious": { + "rev": "ed265cfdcd131b9df5398d899cc5d7036a5b7846", + "sha256": "0hhnq4s0slwbgxra7gxapl7dcz60a7k71cndi4crqcikmazzac3b", + "version": "unstable-2022-03-16" + }, + "lsquic": { + "sha256": "sha256-hG8cUvhbCNeMOsKkaJlgGpzUrIx47E/WhmPIdI5F3qM=", + "version": "2.18.1" + }, + "videojs": { + "sha256": "0b4vxd29kpvy60yhqm376r1872gds17s6wljqw0zlr16j762k50r" + } +} diff --git a/pkgs/servers/invidious/videojs.nix b/pkgs/servers/invidious/videojs.nix new file mode 100644 index 0000000000000..e4470793e51ef --- /dev/null +++ b/pkgs/servers/invidious/videojs.nix @@ -0,0 +1,18 @@ +{ stdenvNoCC, cacert, crystal, openssl, pkg-config, invidious }: + +let + versions = builtins.fromJSON (builtins.readFile ./versions.json); +in +stdenvNoCC.mkDerivation { + name = "videojs"; + + inherit (invidious) src; + + builder = ./videojs.sh; + + nativeBuildInputs = [ cacert crystal openssl pkg-config ]; + + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + outputHash = versions.videojs.sha256; +} diff --git a/pkgs/servers/invidious/videojs.sh b/pkgs/servers/invidious/videojs.sh new file mode 100644 index 0000000000000..31609db14eb3b --- /dev/null +++ b/pkgs/servers/invidious/videojs.sh @@ -0,0 +1,9 @@ +source $stdenv/setup + +unpackPhase +cd source +# this helper downloads the videojs files and checks their checksums +# against videojs-dependencies.yml so it should be pure +crystal run scripts/fetch-player-dependencies.cr -- --minified +rm -f assets/videojs/.gitignore +mv assets/videojs "$out" |