From efc26aaa9c302dc4410394aa2cea81da3764e249 Mon Sep 17 00:00:00 2001 From: bluesaxman Date: Mon, 1 Mar 2021 14:31:36 -0700 Subject: [PATCH] added progress for refresh, added Live check, changed from write on quit to write on datachange so that if program dies data is preserved. --- ytbrowser.pl | 77 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 30 deletions(-) diff --git a/ytbrowser.pl b/ytbrowser.pl index 97a53a6..e35a5a5 100755 --- a/ytbrowser.pl +++ b/ytbrowser.pl @@ -12,7 +12,11 @@ use Curses::UI; #$Data::Dumper::Indent = 1; #$Data::Dumper::Maxdepth = 1; -my $cookiesFile = "/home/bluesaxman/.scripts/cookies.txt"; +my $homeDir = "/home/bluesaxman/"; +my $cookiesFile = $homeDir.".scripts/cookies.txt"; +my $cachedFile = $homeDir.".yt_cache"; +my $playlistFile = $homeDir.".yt_play"; +my $debugFile = $homeDir.".yt_last"; my @watchlist = (); my @suggestArray = (); @@ -38,7 +42,7 @@ $cookieCrisp =~ m/\.youtube\.com[^\n]*SAPISID\s([^\n]*)/; #sha1(new Date().getTime() + " " + SAPISID + " " + origin) $sapisidhash = (gettimeofday())[0]."_".sha1_hex((gettimeofday())[0]." ".$1." https://www.youtube.com"); -open(WATCH, "./watchlist.txt") or goto SKIPWATCHLOAD; +open(WATCH, $playlistFile) or goto SKIPWATCHLOAD; while() { my $item = $_; chomp $item; @@ -55,6 +59,7 @@ my %tests = map { (split("=",$_))[1] => 1 } @watchlist; sub addVideos { my @contents = shift; + my $myProgress = shift; for (@{$contents[0]}) { if (exists($_->{'richItemRenderer'})) { @@ -70,31 +75,30 @@ sub addVideos { }; } elsif (exists($_->{'continuationItemRenderer'})) { $recursionWatchdog++; - unless($recursionWatchdog > 100) { sleep 2; return additionalGet($_->{'continuationItemRenderer'}); } + $myProgress->pos($recursionWatchdog); + $myProgress->draw(); + unless($recursionWatchdog > 100) { sleep 2; return additionalGet($_->{'continuationItemRenderer'},$myProgress); } } } } sub firstGet { -# open(FH, "./ytsample"); -# $rawFile = ""; -# while () { $rawFile .= $_ } -# close(FH); - + my $myProgress = shift; $rawFile = `curl -s -b $cookiesFile https://www.youtube.com`; - + $myProgress->draw(); $rawFile =~ m/ytInitialData =([^<]*)/; $jsonDirty = $1; chop $jsonDirty; $json = decode_json $jsonDirty; $visitorID = $json->{'responseContext'}{'webResponseContextExtensionData'}{'ytConfigData'}{'visitorData'}; $visitorID =~ s/%3D/=/g; - addVideos($json->{'contents'}{'twoColumnBrowseResultsRenderer'}{'tabs'}[0]->{'tabRenderer'}{'content'}{'richGridRenderer'}{'contents'}); + addVideos($json->{'contents'}{'twoColumnBrowseResultsRenderer'}{'tabs'}[0]->{'tabRenderer'}{'content'}{'richGridRenderer'}{'contents'},$myProgress); } sub additionalGet { -# We need this token in order to get more suggestions. +# We need these tokens in order to get more suggestions. my $continuationItem = shift; + my $myProgress = shift; $continuation = $continuationItem->{'continuationEndpoint'}{'continuationCommand'}{'token'}; $pageID = (split(/\|/, $json->{'responseContext'}{'mainAppWebResponseContext'}{'datasyncId'}))[0]; $ytclient = $json->{'responseContext'}{'serviceTrackingParams'}[2]->{'params'}[1]->{'value'}; @@ -122,16 +126,10 @@ sub additionalGet { # Not the curl the world wants, but the curl the world needs - # Our test sample data for a continuation -# open(FH2, "./ytsample_continuation"); -# $rawFile = ""; -# while () { $rawFile .= $_; } -# close(FH2); - $rawFile = `curl -s -X POST -H 'content-type: application/json' -H 'authorization: SAPISIDHASH $sapisidhash' -H 'referer: https://www.youtube.com/' -H 'x-goog-authuser: 0' -H 'x-goog-pageid: $pageID' -H 'x-goog-visitor-id: $visitorID' -H 'x-origin: https://www.youtube.com' -H 'x-youtube-client-name: 1' -H 'x-youtube-client-version: $ytclientVersion' -b $cookiesFile -d '$requestJson' https://www.youtube.com/youtubei/v1/browse?key=$apiKey`; $json = decode_json $rawFile; if (exists($json->{'onResponseReceivedActions'})) { - return addVideos($json->{'onResponseReceivedActions'}[0]->{'appendContinuationItemsAction'}{'continuationItems'}); + return addVideos($json->{'onResponseReceivedActions'}[0]->{'appendContinuationItemsAction'}{'continuationItems'},$myProgress); } } @@ -140,18 +138,40 @@ sub loadSugs { for (@videos) { my $title = $_->{title}; my $playtime = $_->{runtime}; + unless($playtime) { $playtime = "Live"; } push(@suggestArray, sprintf('%3.3s.',$counter).sprintf('%8.8s ',$playtime).$title); $counter ++; } } -open(CACHED, "./cachedSug") or goto SKIPCACHED; +open(CACHED, $cachedFile) or goto SKIPCACHED; my $readFile = ""; while () { $readFile .= $_; } close(CACHED); @videos = @{decode_json($readFile)}; SKIPCACHED: +`touch $debugFile`; +`touch $cachedFile`; +`touch $playlistFile`; +open(DEBUG, ">", $debugFile); +open(CACHING, ">", $cachedFile); +open(SAVE, ">", $playlistFile); + +sub sync2files { +truncate(DEBUG,0); +truncate(CACHING,0); +truncate(SAVE,0); + +seek(DEBUG,0,0); +seek(CACHING,0,0); +seek(SAVE,0,0); + +if ($rawFile) { print DEBUG $rawFile; } +print CACHING encode_json(\@videos); +for (@watchlist) { print SAVE $_."\n"; } +} + #################################################### # Now that we have our data from Youtube, lets make # it easy to look at and select. @@ -168,6 +188,7 @@ $suggestions->set_binding( sub { splice(@suggestArray,$suggestions->get_active_id(),1); push(@watchlist, "https://www.youtube.com/watch?v=".$vid->{id}); # $cui->status("https://www.youtube.com/watch?v=".$vid->{id}."\nhas been added to the watchlist. Watchlist is ".(scalar(@watchlist) + 1)." long."); + sync2files(); $suggestions->draw(); $wlist->draw(); }, KEY_ENTER); @@ -175,6 +196,7 @@ $cui->set_binding( sub { my $video = shift(@watchlist); $cui->status("Playing Video, controls will resume once video is compelte"); `mpv --ytdl-raw-options=cookies=$cookiesFile,mark-watched= --terminal=no $video`; + sync2files(); $win->draw(); $suggestions->draw(); $wlist->draw(); @@ -183,12 +205,15 @@ $cui->set_binding( sub { splice(@videos,0,scalar(@videos)); splice(@suggestArray,0,scalar(@suggestArray)); $cui->status("Refreshing from Youtube, please wait..."); - firstGet(); + $recursionWatchdog = 0; + my $progressBar = $win->add('refresh', 'Progressbar', -max => 50, -pos => 0, -y => $suggestions->height() / 2 ); + $progressBar->focus(); + firstGet($progressBar); loadSugs(); + sync2files(); $win->draw(); $suggestions->draw(); $wlist->draw(); - }, "r"); $cui->set_binding( sub { my $vid = $videos[$suggestions->get_active_id()]; @@ -200,15 +225,7 @@ $suggestions->focus(); $cui->mainloop(); #Save leftovers -`touch ./lastPull`; -open(DEBUG, ">", "./lastPull"); -print DEBUG $rawFile; +sync2files(); close(DEBUG); -`touch ./cachedSug`; -open(CACHING, ">", "./cachedSug"); -print CACHING encode_json(\@videos); close(CACHING); -`touch ./watchlist.txt`; -open(SAVE, ">", "./watchlist.txt"); -for (@watchlist) { print SAVE $_."\n"; } close(SAVE);