From 257987e3c09c8f8be3031d8cad83048891bba17a Mon Sep 17 00:00:00 2001 From: m5r Date: Thu, 16 Sep 2021 06:56:16 +0800 Subject: [PATCH] backup database to s3 --- app/core/api/cron/daily-backup.ts | 5 ++ app/core/api/cron/monthly-backup.ts | 5 ++ app/core/api/cron/weekly-backup.ts | 5 ++ blitz.config.ts | 5 ++ db/backup.ts | 135 ++++++++++++++++++++++++++++ integrations/ses.ts | 38 ++++++++ package-lock.json | 83 +++++++++++++++++ package.json | 1 + 8 files changed, 277 insertions(+) create mode 100644 app/core/api/cron/daily-backup.ts create mode 100644 app/core/api/cron/monthly-backup.ts create mode 100644 app/core/api/cron/weekly-backup.ts create mode 100644 db/backup.ts create mode 100644 integrations/ses.ts diff --git a/app/core/api/cron/daily-backup.ts b/app/core/api/cron/daily-backup.ts new file mode 100644 index 0000000..3af42af --- /dev/null +++ b/app/core/api/cron/daily-backup.ts @@ -0,0 +1,5 @@ +import { CronJob } from "quirrel/blitz"; + +import backup from "../../../../db/backup"; + +export default CronJob("api/cron/daily-backup", "0 0 * * *", async () => backup("daily")); diff --git a/app/core/api/cron/monthly-backup.ts b/app/core/api/cron/monthly-backup.ts new file mode 100644 index 0000000..43048b9 --- /dev/null +++ b/app/core/api/cron/monthly-backup.ts @@ -0,0 +1,5 @@ +import { CronJob } from "quirrel/blitz"; + +import backup from "../../../../db/backup"; + +export default CronJob("api/cron/monthly-backup", "0 0 1 * *", async () => backup("monthly")); diff --git a/app/core/api/cron/weekly-backup.ts b/app/core/api/cron/weekly-backup.ts new file mode 100644 index 0000000..8d3adc6 --- /dev/null +++ b/app/core/api/cron/weekly-backup.ts @@ -0,0 +1,5 @@ +import { CronJob } from "quirrel/blitz"; + +import backup from "../../../../db/backup"; + +export default CronJob("api/cron/weekly-backup", "0 0 * * 0", async () => backup("weekly")); diff --git a/blitz.config.ts b/blitz.config.ts index fe83b8f..afdb6c0 100644 --- a/blitz.config.ts +++ b/blitz.config.ts @@ -56,6 +56,11 @@ const { SENTRY_DSN, SENTRY_ORG, SENTRY_PROJECT, SENTRY_AUTH_TOKEN, NODE_ENV, GIT secretAccessKey: process.env.AWS_SES_ACCESS_KEY_SECRET, fromEmail: process.env.AWS_SES_FROM_EMAIL, }, + awsS3: { + awsRegion: process.env.AWS_S3_REGION, + accessKeyId: process.env.AWS_S3_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_S3_ACCESS_KEY_SECRET, + }, mailChimp: { apiKey: process.env.MAILCHIMP_API_KEY, audienceId: process.env.MAILCHIMP_AUDIENCE_ID, diff --git a/db/backup.ts b/db/backup.ts new file mode 100644 index 0000000..7b267d1 --- /dev/null +++ b/db/backup.ts @@ -0,0 +1,135 @@ +import url from "url"; +import querystring from "querystring"; +import { spawn } from "child_process"; +import { PassThrough } from "stream"; +import { getConfig } from "blitz"; +import AWS from "aws-sdk"; +import { sendEmail } from "../integrations/ses"; + +const { serverRuntimeConfig } = getConfig(); + +const s3 = new AWS.S3({ + credentials: new AWS.Credentials({ + accessKeyId: serverRuntimeConfig.awsS3.accessKeyId, + secretAccessKey: serverRuntimeConfig.awsS3.secretAccessKey, + }), + region: serverRuntimeConfig.awsS3.region, +}); + +export default async function backup(schedule: "daily" | "weekly" | "monthly") { + const s3Bucket = `shellphone-${schedule}-backup`; + const { database, host, port, user, password } = parseDatabaseUrl(process.env.DATABASE_URL!); + const fileName = getFileName(database); + + console.log(`Dumping database ${database}`); + const pgDumpChild = spawn("pg_dump", [`-U${user}`, `-d${database}`], { + env: { + ...process.env, + PGPASSWORD: password, + PGHOST: host, + PGPORT: port.toString(), + }, + stdio: ["ignore", "pipe", "inherit"], + }); + + console.log(`Compressing dump "${fileName}"`); + const gzippedDumpStream = new PassThrough(); + const gzipChild = spawn("gzip", { stdio: ["pipe", "pipe", "inherit"] }); + gzipChild.on("exit", (code) => { + if (code !== 0) { + return sendEmail({ + body: `${schedule} backup failed: gzip: Bad exit code (${code})`, + subject: `${schedule} backup failed: gzip: Bad exit code (${code})`, + recipients: ["error@shellphone.app"], + }); + } + }); + pgDumpChild.stdout.pipe(gzipChild.stdin); + gzipChild.stdout.pipe(gzippedDumpStream); + + pgDumpChild.on("exit", (code) => { + if (code !== 0) { + console.log("pg_dump failed, upload aborted"); + return sendEmail({ + body: `${schedule} backup failed: pg_dump: Bad exit code (${code})`, + subject: `${schedule} backup failed: pg_dump: Bad exit code (${code})`, + recipients: ["error@shellphone.app"], + }); + } + + console.log(`Uploading "${fileName}" to S3 bucket "${s3Bucket}"`); + const uploadPromise = s3 + .upload({ + Bucket: s3Bucket, + Key: fileName, + ACL: "private", + ContentType: "text/plain", + ContentEncoding: "gzip", + Body: gzippedDumpStream, + }) + .promise(); + + uploadPromise + .then(() => console.log(`Successfully uploaded "${fileName}"`)) + .catch((error) => + sendEmail({ + body: `${schedule} backup failed: ${error}`, + subject: `${schedule} backup failed: ${error}`, + recipients: ["error@shellphone.app"], + }), + ); + }); +} + +function getFileName(database: string) { + const now = new Date(); + const year = now.getUTCFullYear(); + const month = (now.getUTCMonth() + 1).toString().padStart(2, "0"); + const day = now.getUTCDate(); + const hours = now.getUTCHours(); + const minutes = now.getUTCMinutes(); + const seconds = now.getUTCSeconds(); + + return `${database}-${year}-${month}-${day}_${hours}-${minutes}-${seconds}.sql.gz`; // 2021-09-15_16-00-02.sql.gz +} + +type DatabaseUrl = { + readonly user: string; + readonly password: string; + readonly host: string; + readonly port: number; + readonly database: string; +}; + +function parseDatabaseUrl(databaseUrl: string): DatabaseUrl { + const parsedUrl = url.parse(databaseUrl, false, true); + const config = querystring.parse(parsedUrl.query!); + + if (parsedUrl.auth) { + const userPassword = parsedUrl.auth.split(":", 2); + config.user = userPassword[0]; + if (userPassword.length > 1) { + config.password = userPassword[1]; + } + } + + if (parsedUrl.pathname) { + config.database = parsedUrl.pathname.replace(/^\//, "").replace(/\/$/, ""); + } + + if (parsedUrl.hostname) { + config.host = parsedUrl.hostname; + } + + if (parsedUrl.port) { + config.port = parsedUrl.port; + } + + return { + user: config.user as string, + password: config.password as string, + host: config.host as string, + port: Number.parseInt(config.port as string, 10), + database: config.database as string, + }; +} diff --git a/integrations/ses.ts b/integrations/ses.ts new file mode 100644 index 0000000..93b5aa9 --- /dev/null +++ b/integrations/ses.ts @@ -0,0 +1,38 @@ +import type { SendEmailRequest } from "aws-sdk/clients/ses"; +import { Credentials, SES } from "aws-sdk"; +import { getConfig } from "blitz"; + +const { serverRuntimeConfig } = getConfig(); + +const credentials = new Credentials({ + accessKeyId: serverRuntimeConfig.awsSes.accessKeyId, + secretAccessKey: serverRuntimeConfig.awsSes.secretAccessKey, +}); +const ses = new SES({ region: serverRuntimeConfig.awsSes.awsRegion, credentials }); + +type SendEmailParams = { + body: string; + subject: string; + recipients: string[]; +}; + +export async function sendEmail({ body, subject, recipients }: SendEmailParams) { + const request: SendEmailRequest = { + Destination: { ToAddresses: recipients }, + Message: { + Body: { + Text: { + Charset: "UTF-8", + Data: body, + }, + }, + Subject: { + Charset: "UTF-8", + Data: subject, + }, + }, + Source: serverRuntimeConfig.awsSes.fromEmail, + }; + + await ses.sendEmail(request).promise(); +} diff --git a/package-lock.json b/package-lock.json index 33f3095..eaf766d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5449,6 +5449,68 @@ "resolved": "https://registry.npmjs.org/awesome-phonenumber/-/awesome-phonenumber-2.58.0.tgz", "integrity": "sha512-rsbIn7Htq/QqUfJ7E53oGiGnLca5SUJEshg8zG5h9WK+fTxoGA12/NDKC5eCvkK2eaP8gR/RVA1yuf0Arib7vg==" }, + "aws-sdk": { + "version": "2.985.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.985.0.tgz", + "integrity": "sha512-Al1oFENrrDeKRpxlklk5sONqzCgEkrhaJ1vtIfpLYYqhNlAY+ku/z1hG1+qSlvgmljGyn7T6/zAb2EcbbAFZLQ==", + "requires": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + }, + "dependencies": { + "buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + } + } + }, "axe-core": { "version": "4.3.3", "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.3.3.tgz", @@ -17611,6 +17673,11 @@ } } }, + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, "saxes": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz", @@ -20378,6 +20445,22 @@ "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==" }, + "xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + }, + "dependencies": { + "xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" + } + } + }, "xmlbuilder": { "version": "13.0.2", "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-13.0.2.tgz", diff --git a/package.json b/package.json index 275d116..96e67bc 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "@tailwindcss/typography": "0.4.1", "@twilio/voice-sdk": "2.0.1", "awesome-phonenumber": "2.58.0", + "aws-sdk": "2.985.0", "blitz": "0.40.0-canary.7", "clsx": "1.1.1", "got": "11.8.2",