diff options
author | illustris <rharikrishnan95@gmail.com> | 2022-03-28 23:18:11 +0530 |
---|---|---|
committer | illustris <rharikrishnan95@gmail.com> | 2022-04-18 02:02:43 +0530 |
commit | 74c684c5b3daf8b6bf79ecd30b3ca27631a4ab79 (patch) | |
tree | b154299441f694035f19e23ad2b938813f676c37 /pkgs/applications/networking/cluster/spark | |
parent | ba88a5afa6fff7710c17b5423ff9d721386c4164 (diff) |
hadoop and spark: improve interoperability
Diffstat (limited to 'pkgs/applications/networking/cluster/spark')
-rw-r--r-- | pkgs/applications/networking/cluster/spark/default.nix | 8 |
1 files changed, 6 insertions, 2 deletions
diff --git a/pkgs/applications/networking/cluster/spark/default.nix b/pkgs/applications/networking/cluster/spark/default.nix index 057be75aeb343..a3ca42dff3424 100644 --- a/pkgs/applications/networking/cluster/spark/default.nix +++ b/pkgs/applications/networking/cluster/spark/default.nix @@ -6,6 +6,7 @@ , python3Packages , extraPythonPackages ? [ ] , coreutils +, hadoopSupport ? true , hadoop , RSupport ? true , R @@ -17,12 +18,13 @@ let spark = { pname, version, sha256, extraMeta ? {} }: stdenv.mkDerivation rec { inherit pname version; + jdk = if hadoopSupport then hadoop.jdk else jdk8; src = fetchzip { url = "mirror://apache/spark/${pname}-${version}/${pname}-${version}-bin-without-hadoop.tgz"; sha256 = sha256; }; nativeBuildInputs = [ makeWrapper ]; - buildInputs = [ jdk8 python3Packages.python ] + buildInputs = [ jdk python3Packages.python ] ++ extraPythonPackages ++ optional RSupport R; @@ -34,9 +36,11 @@ let cp $out/lib/${untarDir}/conf/log4j.properties{.template,} cat > $out/lib/${untarDir}/conf/spark-env.sh <<- EOF - export JAVA_HOME="${jdk8}" + export JAVA_HOME="${jdk}" export SPARK_HOME="$out/lib/${untarDir}" + '' + optionalString hadoopSupport '' export SPARK_DIST_CLASSPATH=$(${hadoop}/bin/hadoop classpath) + '' + '' export PYSPARK_PYTHON="${python3Packages.python}/bin/${python3Packages.python.executable}" export PYTHONPATH="\$PYTHONPATH:$PYTHONPATH" ${optionalString RSupport '' |