Import a single Gist by ID (#76)

server/package.json

@@ -47,6 +47,7 @@
 		"@types/jsonwebtoken": "8.5.8",
 		"@types/marked": "4.0.3",
 		"@types/node": "17.0.21",
+		"@types/node-fetch": "2.6.1",
 		"@types/react-dom": "17.0.15",
 		"@types/supertest": "2.0.12",
 		"cross-env": "7.0.3",

server/src/lib/gist/__tests__/index.ts

@@ -0,0 +1,126 @@
+import { Post } from "@lib/models/Post"
+import { User } from "@lib/models/User"
+import { File } from "@lib/models/File"
+import { Sequelize } from "sequelize-typescript"
+import { createPostFromGist, responseToGist  } from ".."
+import { GistResponse } from "../fetch"
+import { AdditionalPostInformation } from "../transform"
+import * as path from "path"
+
+let aUser: User
+
+let sequelize: Sequelize
+
+beforeAll(async () => {
+	sequelize = new Sequelize({
+		dialect: "sqlite",
+		storage: ":memory:",
+		models: [path.resolve(__dirname, "../../models")],
+		logging: false
+	})
+	await sequelize.authenticate()
+	await sequelize.sync({ force: true })
+
+	aUser = await User.create({
+		username: "a user",
+		password: "monkey",
+		role: "user"
+	})
+})
+
+afterAll(async () => {
+	await sequelize.close()
+})
+
+async function createPost(
+	response: GistResponse,
+	override: Partial<AdditionalPostInformation> = {}
+): Promise<Post> {
+	const info: AdditionalPostInformation = {
+		userId: aUser.id,
+		visibility: "public",
+		...override
+	}
+	return createPostFromGist(info, responseToGist (response))
+}
+
+describe("Gist", () => {
+	it("should fail if the gist has too many files", () => {
+		const tooManyFiles: GistResponse = {
+			id: "some id",
+			created_at: "2022-04-05T18:23:31Z",
+			description: "many files",
+			files: {
+				//... many many files
+			},
+			truncated: true
+		}
+
+		expect(createPost(tooManyFiles)).rejects.toEqual(
+			new Error("Gist has too many files to import")
+		)
+	})
+
+	it("should fail if the gist has no files", () => {
+		const noFiles: GistResponse = {
+			id: "some id",
+			created_at: "2022-04-05T18:23:31Z",
+			description: "no files",
+			files: {},
+			truncated: false
+		}
+
+		expect(createPost(noFiles)).rejects.toEqual(
+			new Error("The gist did not have any files")
+		)
+	})
+
+	it("should create a post for the user with all the files", async () => {
+		const noFiles: GistResponse = {
+			id: "some id",
+			created_at: "2022-04-05T18:23:31Z",
+			description: "This is a gist",
+			files: {
+				"README.md": {
+					content: "this is a readme",
+					filename: "README.md",
+					raw_url: "http://some.url",
+					truncated: false
+				}
+			},
+			truncated: false
+		}
+		const expiresAt = new Date("2022-04-25T18:23:31Z")
+		const newPost = await createPost(noFiles, {
+			password: "password",
+			visibility: "protected",
+			expiresAt
+		})
+
+		const post = await Post.findByPk(newPost.id, {
+			include: [
+				{ model: File, as: "files" },
+				{ model: User, as: "users" }
+			]
+		})
+
+		expect(post).not.toBeNull()
+		expect(post!.title).toBe("This is a gist")
+		expect(post!.visibility).toBe("protected")
+		expect(post!.password).toBe("password")
+		expect(post!.expiresAt!.getDate()).toBe(expiresAt.getDate())
+		expect(post!.createdAt.getDate()).toBe(
+			new Date("2022-04-05T18:23:31Z").getDate()
+		)
+
+		expect(post!.files).toHaveLength(1)
+		expect(post!.files![0].title).toBe("README.md")
+		expect(post!.files![0].content).toBe("this is a readme")
+
+		expect(post!.users).toContainEqual(
+			expect.objectContaining({
+				PostAuthor: expect.objectContaining({ userId: aUser.id })
+			})
+		)
+	})
+})

server/src/lib/gist/fetch.ts

@@ -0,0 +1,71 @@
+import fetch from "node-fetch"
+import { Response } from "node-fetch"
+import { Gist, GistFile } from "./types"
+
+async function fetchHelper(response: Response): Promise<Response> {
+	if (!response.ok) {
+		const isJson = response.headers
+			.get("content-type")
+			?.includes("application/json")
+		const err = await (isJson ? response.json() : response.text())
+		throw new Error(err)
+	}
+	return response
+}
+
+type Timestamp = string // e.g.: "2010-04-14T02:15:15Z"
+interface File {
+	filename: string
+	content: string
+	raw_url: string
+	truncated: boolean
+}
+
+export interface GistResponse {
+	id: string
+	created_at: Timestamp
+	description: String
+	files: {
+		[key: string]: File
+	}
+	truncated: boolean
+}
+
+function toFile(file: File): GistFile {
+	return {
+		filename: file.filename,
+		content: file.truncated
+			? () =>
+					fetch(file.raw_url)
+						.then(fetchHelper)
+						.then((res) => res.text())
+			: () => Promise.resolve(file.content)
+	}
+}
+
+export function responseToGist(response: GistResponse): Gist {
+	if (response.truncated) throw new Error("Gist has too many files to import")
+
+	return {
+		id: response.id,
+		created_at: new Date(response.created_at),
+		description: response.description || Object.keys(response.files)[0],
+		files: Object.values(response.files).map(toFile)
+	}
+}
+
+export async function getGist(id: string): Promise<Gist> {
+	const response: GistResponse = await fetch(
+		`https://api.github.com/gists/${id}`,
+		{
+			method: "GET",
+			headers: {
+				Accept: "application/vnd.github.v3+json"
+			}
+		}
+	)
+		.then(fetchHelper)
+		.then((res) => res.json())
+
+	return responseToGist(response)
+}

server/src/lib/gist/index.ts

@@ -0,0 +1,2 @@
+export { getGist, responseToGist  } from "@lib/gist/fetch"
+export { createPostFromGist } from "@lib/gist/transform"

server/src/lib/gist/transform.ts

@@ -0,0 +1,65 @@
+import getHtmlFromFile from "@lib/get-html-from-drift-file"
+import { Post } from "@lib/models/Post"
+import { File } from "@lib/models/File"
+import { Gist } from "./types"
+import * as crypto from "crypto"
+
+export type AdditionalPostInformation = Pick<
+	Post,
+	"visibility" | "password" | "expiresAt"
+> & {
+	userId: string
+}
+
+export async function createPostFromGist(
+	{ userId, visibility, password, expiresAt }: AdditionalPostInformation,
+	gist: Gist
+): Promise<Post> {
+	const files = Object.values(gist.files)
+	const [title, description] = gist.description.split("\n", 1)
+
+	if (files.length === 0) {
+		throw new Error("The gist did not have any files")
+	}
+
+	const newPost = new Post({
+		title,
+		description,
+		visibility,
+		password,
+		expiresAt,
+		createdAt: new Date(gist.created_at)
+	})
+
+	await newPost.save()
+	await newPost.$add("users", userId)
+	const newFiles = await Promise.all(
+		files.map(async (file) => {
+			const content = await file.content()
+			const html = getHtmlFromFile({ content, title: file.filename })
+			const newFile = new File({
+				title: file.filename,
+				content,
+				sha: crypto
+					.createHash("sha256")
+					.update(content)
+					.digest("hex")
+					.toString(),
+				html: html || "",
+				userId: userId,
+				postId: newPost.id
+			})
+			await newFile.save()
+			return newFile
+		})
+	)
+
+	await Promise.all(
+		newFiles.map(async (file) => {
+			await newPost.$add("files", file.id)
+			await newPost.save()
+		})
+	)
+
+	return newPost
+}

server/src/lib/gist/types.d.ts

@@ -0,0 +1,11 @@
+export interface GistFile {
+	filename: string
+	content: () => Promise<string>
+}
+
+export interface Gist {
+	id: string
+	created_at: Date
+	description: String
+	files: GistFile[]
+}

server/src/routes/posts.ts

@@ -9,6 +9,7 @@ import secretKey from "@lib/middleware/secret-key"
 import { Op } from "sequelize"
 import { PostAuthor } from "@lib/models/PostAuthor"
 import getHtmlFromFile from "@lib/get-html-from-drift-file"
+import { getGist, createPostFromGist } from "@lib/gist"
 
 export const posts = Router()
 
@@ -468,3 +469,45 @@ posts.put(
 		}
 	}
 )
+
+
+posts.post(
+	"/import/gist/id/:id",
+	jwt,
+	celebrate({
+		body: {
+			visibility: Joi.string()
+				.custom(postVisibilitySchema, "valid visibility")
+				.required(),
+			password: Joi.string().optional(),
+			expiresAt: Joi.date().optional().allow(null, "")
+		}
+	}),
+	async (req: UserJwtRequest, res, next) => {
+		try {
+			const { id } = req.params
+			const { visibility, password, expiresAt } = req.body
+			const gist = await getGist(id)
+
+			let hashedPassword: string = ""
+			if (visibility === "protected") {
+				hashedPassword = crypto
+					.createHash("sha256")
+					.update(password)
+					.digest("hex")
+			}
+			const newFile = await createPostFromGist(
+				{
+					userId: req.user!.id,
+					visibility,
+					password: hashedPassword,
+					expiresAt
+				},
+				gist
+			)
+			return res.json(newFile)
+		} catch (e) {
+			res.status(400).json({ error: e.toString() })
+		}
+	}
+)

server/yarn.lock

@@ -804,6 +804,14 @@
   resolved "https://registry.yarnpkg.com/@types/ms/-/ms-0.7.31.tgz#31b7ca6407128a3d2bbc27fe2d21b345397f6197"
   integrity sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==
 
+"@types/node-fetch@2.6.1":
+  version "2.6.1"
+  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.1.tgz#8f127c50481db65886800ef496f20bbf15518975"
+  integrity sha512-oMqjURCaxoSIsHSr1E47QHzbmzNR5rK8McHuNb11BOM9cHcIK3Avy0s/b2JlXHoQGTYS3NsvWzV1M0iK7l0wbA==
+  dependencies:
+    "@types/node" "*"
+    form-data "^3.0.0"
+
 "@types/node@*", "@types/node@17.0.21":
   version "17.0.21"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.21.tgz#864b987c0c68d07b4345845c3e63b75edd143644"