From 159ff74c75a1f31dcb183568efb21bf6d6f07a73 Mon Sep 17 00:00:00 2001 From: Profpatsch Date: Mon, 9 Dec 2019 01:03:12 +0100 Subject: pkgs/profpatsch/youtube2audiopodcast: youtube playlist to rss Initial code that fetches a youtube playlist (from ID) and converts it to an rss feed. --- pkgs/profpatsch/youtube2audiopodcast/Main.hs | 111 +++++++++++++++++++++++ pkgs/profpatsch/youtube2audiopodcast/default.nix | 94 ++++++++++++++++++- 2 files changed, 201 insertions(+), 4 deletions(-) create mode 100755 pkgs/profpatsch/youtube2audiopodcast/Main.hs (limited to 'pkgs') diff --git a/pkgs/profpatsch/youtube2audiopodcast/Main.hs b/pkgs/profpatsch/youtube2audiopodcast/Main.hs new file mode 100755 index 00000000..98be3867 --- /dev/null +++ b/pkgs/profpatsch/youtube2audiopodcast/Main.hs @@ -0,0 +1,111 @@ +{-# language OverloadedStrings #-} +{-# language RecordWildCards #-} +{-# language NamedFieldPuns #-} +{-# language DeriveGeneric #-} +module Main where + +import Data.Text (Text) +import Text.RSS.Syntax +import Text.Feed.Types (Feed(RSSFeed)) +import Text.Feed.Export (textFeed) + +import qualified Data.Aeson as Json +import Data.Aeson (FromJSON) +import GHC.Generics (Generic) +import qualified Data.ByteString.Lazy as BS + +-- | Info from the config file +data Config = Config + { channelName :: Text + , channelURL :: Text + } + deriving (Show, Generic) +instance FromJSON Config where + +data Channel = Channel + { channelInfo :: ChannelInfo + , channelItems :: [ItemInfo] + } + deriving (Show, Generic) +instance FromJSON Channel where + +-- | Info fetched from the channel +data ChannelInfo = ChannelInfo + + { channelDescription :: Text + , channelLastUpdate :: DateString -- TODO + , channelImage :: Maybe () --RSSImage + } + deriving (Show, Generic) +instance FromJSON ChannelInfo where + +-- | Info of each channel item +data ItemInfo = ItemInfo + { itemTitle :: Text + , itemDescription :: Text + , itemYoutubeLink :: Text + , itemCategory :: Text + , itemTags :: [Text] + , itemURL :: Text + , itemSizeBytes :: Integer + , itemHash :: Text + } + deriving (Show, Generic) +instance FromJSON ItemInfo where + +-- main = print $ textFeed $ RSSFeed $ toRSS +-- (ChannelInfo +-- { channelDescription = "description" +-- , channelLastUpdate = "some date" +-- , channelImage = nullImage "imageURL" "imageTitle" "imageLink" +-- }) +-- [] +main :: IO () +main = do + input <- BS.getContents + let (Right rss) = Json.eitherDecode input + let exConfig = (Config + { channelName = "channel name" + , channelURL = "channel url" + }) + print $ textFeed $ RSSFeed $ toRSS exConfig rss + +toRSS :: Config -> Channel -> RSS +toRSS Config{..} Channel{channelInfo, channelItems} = + let ChannelInfo{..} = channelInfo in + (nullRSS channelName channelURL) + { rssChannel = (nullChannel channelName channelURL) + { rssDescription = channelDescription + , rssLastUpdate = Just channelLastUpdate + , rssImage = Nothing --channelImage TODO + , rssGenerator = Just "youtube2audiopodcast" + , rssItems = map rssItem channelItems + } + } + + where + rssItem ItemInfo{..} = (nullItem itemTitle) + { rssItemLink = Just itemYoutubeLink + , rssItemDescription = Just itemDescription + -- rssItemAuthor = + , rssItemCategories = + map (\name -> RSSCategory + { rssCategoryDomain = Nothing + , rssCategoryAttrs = [] + , rssCategoryValue = name + }) $ itemCategory : itemTags + , rssItemEnclosure = Just $ RSSEnclosure + { rssEnclosureURL = itemURL + , rssEnclosureLength = Just itemSizeBytes + , rssEnclosureType = "audio/opus" + , rssEnclosureAttrs = [] + } + , rssItemGuid = Just $ RSSGuid + { rssGuidPermanentURL = Just False + , rssGuidAttrs = [] + , rssGuidValue = itemHash + } + -- TODO: date conversion + -- https://tools.ietf.org/html/rfc822#section-5.1 + -- , rssItemPubDate + } diff --git a/pkgs/profpatsch/youtube2audiopodcast/default.nix b/pkgs/profpatsch/youtube2audiopodcast/default.nix index ed8ad967..f0c9189d 100644 --- a/pkgs/profpatsch/youtube2audiopodcast/default.nix +++ b/pkgs/profpatsch/youtube2audiopodcast/default.nix @@ -8,16 +8,48 @@ let // getBins pkgs.execline [ "fdmove" "backtick" "importas" "if" "redirfd" "pipeline" ] // getBins pkgs.s6-portable-utils [ { use = "s6-cat"; as = "cat"; } - ]; + ] + // getBins pkgs.jl [ "jl" ]; + # fetch the audio of a youtube video to ./audio.opus, given video ID youtube-dl-audio = writeExecline "youtube-dl-audio" { readNArgs = 1; } [ bins.youtube-dl "--verbose" "--extract-audio" "--audio-format" "opus" - "--output" "./audio.opus" "https://www.youtube.com/watch?v=\${1}" + # We have to give a specific filename (with the right extension). + # youtube-dl is really finicky with output filenames. + "--output" "./audio.opus" + "https://www.youtube.com/watch?v=\${1}" ]; + # print youtube playlist information to stdout, given playlist ID + youtube-playlist-info = writeExecline "youtube-playlist-info" { readNArgs = 1; } [ + bins.youtube-dl + "--verbose" + # don’t query detailed info of every video, + # which takes a lot of time + "--flat-playlist" + # print a single line of json to stdout + "--dump-single-json" + "--yes-playlist" + "https://www.youtube.com/playlist?list=\${1}" + ]; + + writeHaskellInterpret = nameOrPath: { withPackages ? lib.const [] }: content: + let ghc = pkgs.haskellPackages.ghcWithPackages withPackages; in + pkgs.writers.makeScriptWriter { + interpreter = "${ghc}/bin/runhaskell"; + check = pkgs.writers.writeDash "ghc-typecheck" '' + ln -s "$1" ./Main.hs + ${ghc}/bin/ghc -fno-code -Wall ./Main.hs + ''; + } nameOrPath content; + + printFeed = writeHaskellInterpret "print-feed" { + withPackages = hps: [ hps.feed hps.aeson ]; + } ./Main.hs; + # minimal CGI request parser for use as UCSPI middleware yolo-cgi = pkgs.writers.writePython3 "yolo-cgi" {} '' import sys @@ -43,6 +75,7 @@ let os.execlp(cmd, cmd, *args) ''; + # print the contents of an envar to the stdin of $@ envvar-to-stdin = writeExecline "envvar-to-stdin" { readNArgs = 1; } [ "importas" "VAR" "$1" "pipeline" [ bins.printf "%s" "$VAR" ] "$@" @@ -86,6 +119,59 @@ let serve-http-opus-file "./audio.opus" ]; + example-config = pkgs.writeText "example-config.json" (lib.generators.toJSON {} { + channelName = "Lonely Rolling Star"; + channelURL = "https://www.youtube.com/playlist?list=PLV9hywkogVcOuHJ8O121ulSfFDKUhJw66"; + }); + + transform-flat-playlist-to-rss = hostUrl: + let + playlist-item-info-jl = '' + (\o -> + { itemTitle: o.title + , itemYoutubeLink: append "https://youtube.com/watch?v=" o.id + ${/*TODO how to add the url here nicely?*/""} + , itemURL: append "${hostUrl}/" o.id + + ${/*# TODO*/""} + , itemDescription: "" + , itemCategory: "" + , itemTags: [] + , itemSizeBytes: 0 + , itemHash: "" + }) + ''; + playlist-info-jl = '' + \pl -> + { channelInfo: + { channelDescription: pl.title + + ${/*# TODO*/""} + , channelLastUpdate: "000" + , channelImage: null + } + , channelItems: + map + ${playlist-item-info-jl} + pl.entries + } + ''; + in writeExecline "youtube-dl-playlist-json-to-rss-json" {} [ + bins.jl playlist-info-jl + ]; + + print-feed-json = writeExecline "ex2" {} [ + "pipeline" [ + youtube-playlist-info "PLV9hywkogVcOuHJ8O121ulSfFDKUhJw66" + ] (transform-flat-playlist-to-rss "localhost:8888") + ]; + + print-example-feed = writeExecline "ex" {} [ + "pipeline" [ print-feed-json ] + printFeed + ]; + + # in printFeed -in serve-audio -# in youtube-dl-audio +# in serve-audio +in print-example-feed -- cgit 1.4.1