fork/cohost-backup

git clone https://git.julia.blue/fork/cohost-backup

G-F-Dsupport 'attachment-row' content type3c0dbdc

main
9.2 KiB261 linesraw
1#!/usr/bin/env pwsh
2
3param (
4	[Parameter(Mandatory)]
5	[string]$username
6)
7
8$ErrorActionPreference = "Stop"
9$PSNativeCommandUseErrorActionPreference = $true
10Set-StrictMode -Version Latest
11
12# Optional chaining, like `?.` in TypeScript
13function Select-Property() {
14	param(
15		[Parameter(ValueFromPipeline)]
16		$obj,
17		[Parameter(Position, Mandatory)]
18		$prop
19	)
20    if ($obj -eq $null) { return $null }
21	# in powershell 5 `select-object` doesn't work on hashmaps :(
22	if ($obj.gettype() -eq [hashtable]) { return $obj[$prop] }
23	($obj | Select-Object $prop).$prop
24}
25
26# COALESCE
27function Skip-Null() {
28	param(
29		[Parameter(ValueFromPipeline)]
30		$obj,
31		[Parameter(Position, Mandatory)]
32		$default
33	)
34	if ($obj -eq $null) { $default } else { $obj }
35}
36
37Set-Alias '?.' Select-Property
38Set-Alias '??' Skip-Null
39if (Test-Path Alias:curl) { Remove-Item Alias:curl }
40# not a typo https://stackoverflow.com/a/24743647/7669110
41if (Test-Path Alias:curl) { Remove-Item Alias:curl }
42
43# Install the PSParseHTML module on demand
44function Install-HtmlParser() {
45	# Avoid a prompt on base windows
46	if (-not (Get-PackageProvider -ListAvailable | Where-Object Name -eq NuGet)) {
47		Install-PackageProvider -Scope CurrentUser -Force -name NuGet
48	}
49	if (-not (Get-Module -ErrorAction Ignore -ListAvailable PSParseHTML)) {
50		Install-Module -Scope CurrentUser -Force PSParseHTML
51	}
52}
53
54function download() {
55    # powershell silently strips unknown dash args????
56	$curlArgs = $MyInvocation.UnboundArguments
57	$input | curl --retry 3 --fail $curlArgs
58}
59
60function Get-Posts($file) {
61	$contents = Get-Content -Raw $file
62	$dom = ConvertFrom-Html -Engine AngleSharp -Content $contents
63	$json = ConvertFrom-Json ($dom.QuerySelectorAll('script#trpc-dehydrated-state').TextContent)
64	return $json.queries | %{$_.state | ?. data | ?. posts} | Where-Object { $_ -ne $null }
65}
66
67function Get-Likes($file) {
68	Install-HtmlParser
69	$contents = Get-Content -Raw $file
70	$dom = ConvertFrom-Html -Engine AngleSharp -Content $contents
71	$json = ConvertFrom-Json ($dom.QuerySelectorAll('script#__COHOST_LOADER_STATE__').TextContent)
72	return $json.'liked-posts-feed'.posts | Where-Object { $_ -ne $null }
73}
74
75function Get-TokenMaybe() {
76	$sid = Read-Host -MaskInput "cohost posts are public, but likes are not. to download your liked posts, this script needs to log in as you.
77to get your access token, do the following things:
781. open https://cohost.org/ while logged in.
792. open the devtools network tab. if you are on firefox, press ctrl+shift+e. if you are on chrome or edge, press ctrl+shift+i, then click on the 'Network' tab.
803. on the left panel, click on any request which sends a session cookie to cohost.org ('login.loggedIn,projects.listEditedProjects' is often near the top, that's fine to use).
814. filter for the 'Cookie' header. in firefox, on the right panel, click on 'Filter Headers'. in chrome, press ctrl+f.
825. under 'Request Headers', you should see a string starting with 'Cookie: connect.sid='. right click it and hit 'copy value'.
836. paste that string here.
84if for any reason you don't want this script to log in as you, or you just think that sounds hard and annoying, press Enter now to skip downloading likes.
85
86session cookie (or leave blank to skip downloading liked posts)"
87	if (! $sid) {
88		Write-Host "no token supplied; skipping liked posts"
89		return $null
90	} elseif ($sid.toLower() -in @('set-cookie', 'cookie')) {
91		Write-Error "you copied multiple lines out of devtools; this script needs you to paste exactly one line (in firefox you can right click and use 'Copy Value')"
92	}
93	(($sid -replace '.*connect.sid=([^ ;]*).*', '$1') -split '\n')[-1]
94}
95
96# cohost posts contain 0 or more posts in a chain named "shareTree"; more than 0 indicates this is a quote-post/repost.
97# note that cohost does not really distinguish a repost from a quote post without text
98function Get-ChainContent($chain) {
99	function Get-Project($project) {
100		@{id=$project.projectId; handle=$project.handle; displayName=$project.displayName}
101	}
102	function Get-Post($post) {
103		$content = $post.blocks | %{ $block = $_; switch($block.type) {
104			'markdown' { @{markdown=$block.markdown.content} }
105			'ask' {
106				$ask = $block.ask
107				$who = if ($ask | ?. anon) { @{anon=$ask.anon} } else { Get-Project $ask.askingProject }
108				@{ask=@{content=$ask.content; sentAt=$ask.sentAt; who=$who}}
109			}
110			'attachment' {
111				$attachment = $block.attachment
112				@{img=@{altText=$attachment | ?. altText; fileUrl=$attachment.fileURL}}
113			}
114			'attachment-row' {
115				foreach ($att in $block.attachments) {
116					$attachment = $att.attachment
117					@{img=@{altText=$attachment | ?. altText; fileUrl=$attachment.fileURL}}
118				}
119			}
120			default {
121				Write-Error "unknown content type $($block.type)"
122			}
123		} };
124		@{content=@($content); poster=Get-Project $post.postingProject;
125		  filename=$post.filename; publishedAt=$post.publishedAt;
126		  cws=$post.cws; tags=$post.tags}
127	}
128	$d = Get-Post $chain
129	$d.shareTree = @($chain.shareTree | %{Get-Post $_})
130	return $d
131}
132
133function Format-Time($when) {
134	# TODO: show the abbreviated time zone too
135	([datetime]$when).toLocalTime().toString("yyyy-MM-dd HH:mm:ss")
136}
137
138function Format-WhoWhen($who, $when, $how) {
139	"**$($who.displayName) | $($who.handle)** ${how} at $(Format-Time $when)"
140}
141
142function Format-Post($post) {
143	$rendered = $post.content | %{
144		$keys = $_.keys
145		if ("markdown" -in $keys) {
146			$_.markdown
147		} elseif ("ask" -in $keys) {
148			$ask = $_.ask
149			$content, $sent, $who = $ask.content, $ask.sentAt, $ask.who
150			$who_when = if ($who | ?. anon) {
151				"anon asked at $(Format-Time $sent)"
152			} else {
153				Format-WhoWhen $who $sent "asked"
154			}
155			'${who_when}:
156
157```quote
158' + $content + '
159```'
160		} else {
161			$alt, $url = ($_.img.altText | ?? ""), [uri]$_.img.fileUrl
162			$hash = $url.segments[-2].TrimEnd("/")
163			$ext = [System.IO.Path]::GetExtension($url.segments[-1])
164			# this is a little absurd :( https://stackoverflow.com/a/73391369
165			$dst = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath("../img/${hash}$ext")
166			if (! (Test-Path $dst) -or (Get-Item $dst).length -eq 0) {
167				# note: curl tries to interpret backslashes in configs :(
168				$global:imgs += 'output="' + ($dst -replace "\\", "\\") + '"'
169				# for reasons i don't understand, powershell puts this on the same line when passed to curl if we use $imgs += (output, url), so it has to be a separate statement
170				$global:imgs += 'url='+$url.AbsoluteUri
171			}
172			# markdown doesn't allow newlines in image alt text
173            # TODO: figure out how to actually keep these: https://tech.lgbt/@jyn/112117398042554191
174            $alt = $alt -replace "`n", ""
175			"![$alt](file:///$dst)"
176		}
177	}
178	$tags = $post.tags | %{"#$_"}
179	$descr = if ($post.content) { "said" } else { "reshared this post" }
180	$who_when = Format-WhoWhen $post.poster $post.publishedAt $descr
181	$combined = if ($post.content) { "${who_when}: `n`n" + ($rendered -join "`n`n") } else { $who_when }
182	if ($tags) { $combined += "`n`n" + ($tags -join ", ") }
183	$combined
184}
185
186function Write-Chain($ir) {
187	$rendered = ($ir.shareTree | %{Format-Post $_}) + (Format-Post $ir)
188	$rendered -join "`n`n" > "rendered/$($_.filename).md"
189}
190
191function Get-AllLikes($sid) {
192	# download likes
193	Push-Location ../likes
194	trap { Pop-Location }
195	New-Item -ItemType Directory -ErrorAction SilentlyContinue @('raw', 'parsed', 'rendered') >$null
196	$page = 0
197	$liked = 0
198	while($true) {
199		$html = "raw/${page}.html"
200		$parsed = "parsed/${page}.json"
201		if (! (Test-Path $html) -or (Get-Item $html).length -eq 0) {
202			Write-Output "fetching page $page of likes"
203			download --no-progress-meter "https://cohost.org/rc/liked-posts?skipPosts=$liked" --cookie "connect.sid=$sid" -o $html
204			if ((Get-Item $html | Measure-Object -Line).lines -eq 1 -and (Get-Content $html) -like '*/rc/login*') {
205				Remove-Item $html
206				Write-Error "cohost thinks you're not logged in (are you sure you pasted the right token?)"
207			}
208		}
209		$posts = Get-Likes $html
210		$num_likes = ($posts | Measure-Object).Count
211		if ($num_likes -eq 0) {
212			break  # no posts left
213		}
214		Write-Output "parsing and rendering liked posts starting from page $page"
215		$ir = $posts | %{ Get-ChainContent $_ }
216		$ir | ConvertTo-Json -Depth 100 > $parsed
217		$ir | %{ Write-Chain $_ }
218		$page += 1
219		$liked += $num_likes
220	}
221}
222
223$sid = Get-TokenMaybe
224$global:imgs = @()
225Install-HtmlParser
226
227# download posts
228New-Item -ItemType Directory -ErrorAction SilentlyContinue @('posts', 'likes', 'img') >$null
229Push-Location posts
230trap { Pop-Location }
231New-Item -ItemType Directory -ErrorAction SilentlyContinue @('raw', 'parsed', 'rendered') >$null
232$page = 0
233while($true) {
234	$html = "raw/${page}.html"
235	$parsed = "parsed/${page}.json"
236	if (! (Test-Path $html) -or (Get-Item $html).length -eq 0) {
237		Write-Output "fetch page $page of posts"
238		download --no-progress-meter "https://cohost.org/${username}?page=$page" -o $html
239	}
240	$posts = Get-Posts $html
241	if (($posts | Measure-Object).Count -eq 0) {
242		break  # no posts left
243	}
244	Write-Output "parsing and rendering posts starting from page $page"
245	$ir = $posts | %{ Get-ChainContent $_ }
246	$ir | ConvertTo-Json -Depth 100 > $parsed
247	$ir | %{ Write-Chain $_ }
248	$page += 1
249}
250
251
252if ($sid) {
253	Get-AllLikes $sid
254} else {
255	Write-Warning "no access token provided; can't download liked posts"
256}
257
258if ($global:imgs) {
259	Write-Host "Downloading $($global:imgs.length) images"
260	$global:imgs | download --parallel -K -
261}