fork/cohost-backup
git clone https://git.julia.blue/fork/cohost-backup
3c0dbdc
main
1#!/usr/bin/env pwsh 2 3param ( 4 [Parameter (Mandatory)] 5 [string ]$username 6) 7 8$ErrorActionPreference ="Stop" 9$PSNativeCommandUseErrorActionPreference =$true 10Set-StrictMode -Version Latest 11 12# Optional chaining, like `?.` in TypeScript 13function Select-Property () { 14param ( 15 [Parameter (ValueFromPipeline)] 16$obj , 17 [Parameter (Position, Mandatory)] 18$prop 19 ) 20if ($obj -eq $null ) {return $null } 21# in powershell 5 `select-object` doesn't work on hashmaps :( 22if ($obj .gettype ()-eq [hashtable ]) {return $obj [$prop ] } 23 ($obj |Select-Object $prop ).$prop 24} 25 26# COALESCE 27function Skip-Null () { 28param ( 29 [Parameter (ValueFromPipeline)] 30$obj , 31 [Parameter (Position, Mandatory)] 32$default 33 ) 34if ($obj -eq $null ) {$default }else {$obj } 35} 36 37Set-Alias '?.' Select-Property 38Set-Alias '??' Skip-Null 39if (Test-Path Alias:curl) {Remove-Item Alias:curl } 40# not a typo https://stackoverflow.com/a/24743647/7669110 41if (Test-Path Alias:curl) {Remove-Item Alias:curl } 42 43# Install the PSParseHTML module on demand 44function Install-HtmlParser () { 45# Avoid a prompt on base windows 46if (-not (Get-PackageProvider -ListAvailable |Where-Object Name -eq NuGet)) { 47Install-PackageProvider -Scope CurrentUser -Force -name NuGet 48 } 49if (-not (Get-Module -ErrorAction Ignore -ListAvailable PSParseHTML)) { 50Install-Module -Scope CurrentUser -Force PSParseHTML 51 } 52} 53 54function download () { 55# powershell silently strips unknown dash args???? 56$curlArgs =$MyInvocation .UnboundArguments 57$input |curl --retry3 --fail$curlArgs 58} 59 60function Get-Posts ($file ) { 61$contents =Get-Content -Raw$file 62$dom =ConvertFrom-Html -Engine AngleSharp -Content$contents 63$json =ConvertFrom-Json ($dom .QuerySelectorAll ('script#trpc-dehydrated-state' ).TextContent ) 64return $json .queries |% {$_ .state |?. data |?. posts} |Where-Object {$_ -ne $null } 65} 66 67function Get-Likes ($file ) { 68Install-HtmlParser 69$contents =Get-Content -Raw$file 70$dom =ConvertFrom-Html -Engine AngleSharp -Content$contents 71$json =ConvertFrom-Json ($dom .QuerySelectorAll ('script#__COHOST_LOADER_STATE__' ).TextContent ) 72return $json .'liked-posts-feed' .posts |Where-Object {$_ -ne $null } 73} 74 75function Get-TokenMaybe () { 76$sid =Read-Host -MaskInput"cohost posts are public, but likes are not. to download your liked posts, this script needs to log in as you. 77to get your access token, do the following things: 781. open https://cohost.org/ while logged in. 792. open the devtools network tab. if you are on firefox, press ctrl+shift+e. if you are on chrome or edge, press ctrl+shift+i, then click on the 'Network' tab. 803. on the left panel, click on any request which sends a session cookie to cohost.org ('login.loggedIn,projects.listEditedProjects' is often near the top, that's fine to use). 814. filter for the 'Cookie' header. in firefox, on the right panel, click on 'Filter Headers'. in chrome, press ctrl+f. 825. under 'Request Headers', you should see a string starting with 'Cookie: connect.sid='. right click it and hit 'copy value'. 836. paste that string here. 84if for any reason you don't want this script to log in as you, or you just think that sounds hard and annoying, press Enter now to skip downloading likes. 85 86session cookie (or leave blank to skip downloading liked posts)" 87if (!$sid ) { 88Write-Host "no token supplied; skipping liked posts" 89return $null 90 }elseif ($sid .toLower ()-in @('set-cookie' ,'cookie' )) { 91Write-Error "you copied multiple lines out of devtools; this script needs you to paste exactly one line (in firefox you can right click and use 'Copy Value')" 92 } 93 (($sid -replace '.*connect.sid=([^ ;]*).*' ,'$1' )-split '\n' )[- 1 ] 94} 95 96# cohost posts contain 0 or more posts in a chain named "shareTree"; more than 0 indicates this is a quote-post/repost. 97# note that cohost does not really distinguish a repost from a quote post without text 98function Get-ChainContent ($chain ) { 99function Get-Project ($project ) { 100 @{id=$project .projectId ; handle=$project .handle ; displayName=$project .displayName } 101 } 102function Get-Post ($post ) { 103$content =$post .blocks |% {$block =$_ ;switch ($block .type ) { 104'markdown' { @{markdown=$block .markdown .content } } 105'ask' { 106$ask =$block .ask 107$who =if ($ask |?. anon) { @{anon=$ask .anon } }else {Get-Project $ask .askingProject } 108 @{ask=@{content=$ask .content ; sentAt=$ask .sentAt ; who=$who }} 109 } 110'attachment' { 111$attachment =$block .attachment 112 @{img=@{altText=$attachment |?. altText; fileUrl=$attachment .fileURL }} 113 } 114'attachment-row' { 115foreach ($att in$block .attachments ) { 116$attachment =$att .attachment 117 @{img=@{altText=$attachment |?. altText; fileUrl=$attachment .fileURL }} 118 } 119 } 120 default { 121Write-Error "unknown content type $( $block . type )" 122 } 123 } }; 124 @{content=@($content ); poster=Get-Project $post .postingProject ; 125 filename=$post .filename ; publishedAt=$post .publishedAt ; 126 cws=$post .cws ; tags=$post .tags } 127 } 128$d =Get-Post $chain 129$d .shareTree = @($chain .shareTree |% {Get-Post $_ }) 130return $d 131} 132 133function Format-Time ($when ) { 134# TODO: show the abbreviated time zone too 135 ([datetime ]$when ).toLocalTime ().toString ("yyyy-MM-dd HH:mm:ss" ) 136} 137 138function Format-WhoWhen ($who ,$when ,$how ) { 139"**$( $who . displayName ) | $( $who . handle )** ${how} at $( Format-Time $when )" 140} 141 142function Format-Post ($post ) { 143$rendered =$post .content |% { 144$keys =$_ .keys 145if ("markdown" -in $keys ) { 146$_ .markdown 147 }elseif ("ask" -in $keys ) { 148$ask =$_ .ask 149$content ,$sent ,$who =$ask .content ,$ask .sentAt ,$ask .who 150$who_when =if ($who |?. anon) { 151"anon asked at $( Format-Time $sent )" 152 }else { 153Format-WhoWhen $who $sent "asked" 154 } 155'${who_when}: 156 157```quote 158' + $content + ' 159```' 160 }else { 161$alt ,$url = ($_ .img .altText |?? "" ), [uri ]$_ .img .fileUrl 162$hash =$url .segments [- 2 ].TrimEnd ("/" ) 163$ext = [System.IO.Path ]::GetExtension ($url .segments [- 1 ]) 164# this is a little absurd :( https://stackoverflow.com/a/73391369 165$dst =$ExecutionContext .SessionState .Path .GetUnresolvedProviderPathFromPSPath ("../img/ ${hash}$ext " ) 166if (! (Test-Path $dst )-or (Get-Item $dst ).length -eq 0 ) { 167# note: curl tries to interpret backslashes in configs :( 168$global:imgs +='output="' + ($dst -replace "\\" ,"\\" )+ '"' 169# for reasons i don't understand, powershell puts this on the same line when passed to curl if we use $imgs += (output, url), so it has to be a separate statement 170$global:imgs +='url=' + $url .AbsoluteUri 171 } 172# markdown doesn't allow newlines in image alt text 173# TODO: figure out how to actually keep these: https://tech.lgbt/@jyn/112117398042554191 174$alt =$alt -replace "`n" ,"" 175"" 176 } 177 } 178$tags =$post .tags |% {"# $_ " } 179$descr =if ($post .content ) {"said" }else {"reshared this post" } 180$who_when =Format-WhoWhen $post .poster $post .publishedAt $descr 181$combined =if ($post .content ) {" ${who_when} : `n`n" + ($rendered -join "`n`n" ) }else {$who_when } 182if ($tags ) {$combined +="`n`n" + ($tags -join ", " ) } 183$combined 184} 185 186function Write-Chain ($ir ) { 187$rendered = ($ir .shareTree |% {Format-Post $_ })+ (Format-Post $ir ) 188$rendered -join "`n`n" >"rendered/$( $_ . filename ).md" 189} 190 191function Get-AllLikes ($sid ) { 192# download likes 193Push-Location ../likes 194trap {Pop-Location } 195New-Item -ItemType Directory -ErrorAction SilentlyContinue @('raw' ,'parsed' ,'rendered' ) >$null 196$page =0 197$liked =0 198while ($true ) { 199$html ="raw/ ${page} .html" 200$parsed ="parsed/ ${page} .json" 201if (! (Test-Path $html )-or (Get-Item $html ).length -eq 0 ) { 202Write-Output "fetching page $page of likes" 203download --no-progress-meter"https://cohost.org/rc/liked-posts?skipPosts= $liked " --cookie"connect.sid= $sid " -o$html 204if ((Get-Item $html |Measure-Object -Line).lines -eq 1 -and (Get-Content $html )-like '*/rc/login*' ) { 205Remove-Item $html 206Write-Error "cohost thinks you're not logged in (are you sure you pasted the right token?)" 207 } 208 } 209$posts =Get-Likes $html 210$num_likes = ($posts |Measure-Object ).Count 211if ($num_likes -eq 0 ) { 212break # no posts left 213 } 214Write-Output "parsing and rendering liked posts starting from page $page " 215$ir =$posts |% {Get-ChainContent $_ } 216$ir |ConvertTo-Json -Depth100 >$parsed 217$ir |% {Write-Chain $_ } 218$page +=1 219$liked +=$num_likes 220 } 221} 222 223$sid =Get-TokenMaybe 224$global:imgs = @() 225Install-HtmlParser 226 227# download posts 228New-Item -ItemType Directory -ErrorAction SilentlyContinue @('posts' ,'likes' ,'img' ) >$null 229Push-Location posts 230trap {Pop-Location } 231New-Item -ItemType Directory -ErrorAction SilentlyContinue @('raw' ,'parsed' ,'rendered' ) >$null 232$page =0 233while ($true ) { 234$html ="raw/ ${page} .html" 235$parsed ="parsed/ ${page} .json" 236if (! (Test-Path $html )-or (Get-Item $html ).length -eq 0 ) { 237Write-Output "fetch page $page of posts" 238download --no-progress-meter"https://cohost.org/ ${username} ?page= $page " -o$html 239 } 240$posts =Get-Posts $html 241if (($posts |Measure-Object ).Count -eq 0 ) { 242break # no posts left 243 } 244Write-Output "parsing and rendering posts starting from page $page " 245$ir =$posts |% {Get-ChainContent $_ } 246$ir |ConvertTo-Json -Depth100 >$parsed 247$ir |% {Write-Chain $_ } 248$page +=1 249} 250 251 252if ($sid ) { 253Get-AllLikes $sid 254}else { 255Write-Warning "no access token provided; can't download liked posts" 256} 257 258if ($global:imgs ) { 259Write-Host "Downloading $( $global:imgs . length ) images" 260$global:imgs | download --parallel -K- 261}