backup database to s3
This commit is contained in:
m5r
parent
9ef5b58400
commit
257987e3c0
5
app/core/api/cron/daily-backup.ts
Normal file
5
app/core/api/cron/daily-backup.ts
Normal file
@ -0,0 +1,5 @@
|
||||
import { CronJob } from "quirrel/blitz";
|
||||
|
||||
import backup from "../../../../db/backup";
|
||||
|
||||
export default CronJob("api/cron/daily-backup", "0 0 * * *", async () => backup("daily"));
|
||||
5
app/core/api/cron/monthly-backup.ts
Normal file
5
app/core/api/cron/monthly-backup.ts
Normal file
@ -0,0 +1,5 @@
|
||||
import { CronJob } from "quirrel/blitz";
|
||||
|
||||
import backup from "../../../../db/backup";
|
||||
|
||||
export default CronJob("api/cron/monthly-backup", "0 0 1 * *", async () => backup("monthly"));
|
||||
5
app/core/api/cron/weekly-backup.ts
Normal file
5
app/core/api/cron/weekly-backup.ts
Normal file
@ -0,0 +1,5 @@
|
||||
import { CronJob } from "quirrel/blitz";
|
||||
|
||||
import backup from "../../../../db/backup";
|
||||
|
||||
export default CronJob("api/cron/weekly-backup", "0 0 * * 0", async () => backup("weekly"));
|
||||
@ -56,6 +56,11 @@ const { SENTRY_DSN, SENTRY_ORG, SENTRY_PROJECT, SENTRY_AUTH_TOKEN, NODE_ENV, GIT
|
||||
secretAccessKey: process.env.AWS_SES_ACCESS_KEY_SECRET,
|
||||
fromEmail: process.env.AWS_SES_FROM_EMAIL,
|
||||
},
|
||||
awsS3: {
|
||||
awsRegion: process.env.AWS_S3_REGION,
|
||||
accessKeyId: process.env.AWS_S3_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_S3_ACCESS_KEY_SECRET,
|
||||
},
|
||||
mailChimp: {
|
||||
apiKey: process.env.MAILCHIMP_API_KEY,
|
||||
audienceId: process.env.MAILCHIMP_AUDIENCE_ID,
|
||||
|
||||
135
db/backup.ts
Normal file
135
db/backup.ts
Normal file
@ -0,0 +1,135 @@
|
||||
import url from "url";
|
||||
import querystring from "querystring";
|
||||
import { spawn } from "child_process";
|
||||
import { PassThrough } from "stream";
|
||||
import { getConfig } from "blitz";
|
||||
import AWS from "aws-sdk";
|
||||
import { sendEmail } from "../integrations/ses";
|
||||
|
||||
const { serverRuntimeConfig } = getConfig();
|
||||
|
||||
const s3 = new AWS.S3({
|
||||
credentials: new AWS.Credentials({
|
||||
accessKeyId: serverRuntimeConfig.awsS3.accessKeyId,
|
||||
secretAccessKey: serverRuntimeConfig.awsS3.secretAccessKey,
|
||||
}),
|
||||
region: serverRuntimeConfig.awsS3.region,
|
||||
});
|
||||
|
||||
export default async function backup(schedule: "daily" | "weekly" | "monthly") {
|
||||
const s3Bucket = `shellphone-${schedule}-backup`;
|
||||
const { database, host, port, user, password } = parseDatabaseUrl(process.env.DATABASE_URL!);
|
||||
const fileName = getFileName(database);
|
||||
|
||||
console.log(`Dumping database ${database}`);
|
||||
const pgDumpChild = spawn("pg_dump", [`-U${user}`, `-d${database}`], {
|
||||
env: {
|
||||
...process.env,
|
||||
PGPASSWORD: password,
|
||||
PGHOST: host,
|
||||
PGPORT: port.toString(),
|
||||
},
|
||||
stdio: ["ignore", "pipe", "inherit"],
|
||||
});
|
||||
|
||||
console.log(`Compressing dump "${fileName}"`);
|
||||
const gzippedDumpStream = new PassThrough();
|
||||
const gzipChild = spawn("gzip", { stdio: ["pipe", "pipe", "inherit"] });
|
||||
gzipChild.on("exit", (code) => {
|
||||
if (code !== 0) {
|
||||
return sendEmail({
|
||||
body: `${schedule} backup failed: gzip: Bad exit code (${code})`,
|
||||
subject: `${schedule} backup failed: gzip: Bad exit code (${code})`,
|
||||
recipients: ["error@shellphone.app"],
|
||||
});
|
||||
}
|
||||
});
|
||||
pgDumpChild.stdout.pipe(gzipChild.stdin);
|
||||
gzipChild.stdout.pipe(gzippedDumpStream);
|
||||
|
||||
pgDumpChild.on("exit", (code) => {
|
||||
if (code !== 0) {
|
||||
console.log("pg_dump failed, upload aborted");
|
||||
return sendEmail({
|
||||
body: `${schedule} backup failed: pg_dump: Bad exit code (${code})`,
|
||||
subject: `${schedule} backup failed: pg_dump: Bad exit code (${code})`,
|
||||
recipients: ["error@shellphone.app"],
|
||||
});
|
||||
}
|
||||
|
||||
console.log(`Uploading "${fileName}" to S3 bucket "${s3Bucket}"`);
|
||||
const uploadPromise = s3
|
||||
.upload({
|
||||
Bucket: s3Bucket,
|
||||
Key: fileName,
|
||||
ACL: "private",
|
||||
ContentType: "text/plain",
|
||||
ContentEncoding: "gzip",
|
||||
Body: gzippedDumpStream,
|
||||
})
|
||||
.promise();
|
||||
|
||||
uploadPromise
|
||||
.then(() => console.log(`Successfully uploaded "${fileName}"`))
|
||||
.catch((error) =>
|
||||
sendEmail({
|
||||
body: `${schedule} backup failed: ${error}`,
|
||||
subject: `${schedule} backup failed: ${error}`,
|
||||
recipients: ["error@shellphone.app"],
|
||||
}),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function getFileName(database: string) {
|
||||
const now = new Date();
|
||||
const year = now.getUTCFullYear();
|
||||
const month = (now.getUTCMonth() + 1).toString().padStart(2, "0");
|
||||
const day = now.getUTCDate();
|
||||
const hours = now.getUTCHours();
|
||||
const minutes = now.getUTCMinutes();
|
||||
const seconds = now.getUTCSeconds();
|
||||
|
||||
return `${database}-${year}-${month}-${day}_${hours}-${minutes}-${seconds}.sql.gz`; // 2021-09-15_16-00-02.sql.gz
|
||||
}
|
||||
|
||||
type DatabaseUrl = {
|
||||
readonly user: string;
|
||||
readonly password: string;
|
||||
readonly host: string;
|
||||
readonly port: number;
|
||||
readonly database: string;
|
||||
};
|
||||
|
||||
function parseDatabaseUrl(databaseUrl: string): DatabaseUrl {
|
||||
const parsedUrl = url.parse(databaseUrl, false, true);
|
||||
const config = querystring.parse(parsedUrl.query!);
|
||||
|
||||
if (parsedUrl.auth) {
|
||||
const userPassword = parsedUrl.auth.split(":", 2);
|
||||
config.user = userPassword[0];
|
||||
if (userPassword.length > 1) {
|
||||
config.password = userPassword[1];
|
||||
}
|
||||
}
|
||||
|
||||
if (parsedUrl.pathname) {
|
||||
config.database = parsedUrl.pathname.replace(/^\//, "").replace(/\/$/, "");
|
||||
}
|
||||
|
||||
if (parsedUrl.hostname) {
|
||||
config.host = parsedUrl.hostname;
|
||||
}
|
||||
|
||||
if (parsedUrl.port) {
|
||||
config.port = parsedUrl.port;
|
||||
}
|
||||
|
||||
return {
|
||||
user: config.user as string,
|
||||
password: config.password as string,
|
||||
host: config.host as string,
|
||||
port: Number.parseInt(config.port as string, 10),
|
||||
database: config.database as string,
|
||||
};
|
||||
}
|
||||
38
integrations/ses.ts
Normal file
38
integrations/ses.ts
Normal file
@ -0,0 +1,38 @@
|
||||
import type { SendEmailRequest } from "aws-sdk/clients/ses";
|
||||
import { Credentials, SES } from "aws-sdk";
|
||||
import { getConfig } from "blitz";
|
||||
|
||||
const { serverRuntimeConfig } = getConfig();
|
||||
|
||||
const credentials = new Credentials({
|
||||
accessKeyId: serverRuntimeConfig.awsSes.accessKeyId,
|
||||
secretAccessKey: serverRuntimeConfig.awsSes.secretAccessKey,
|
||||
});
|
||||
const ses = new SES({ region: serverRuntimeConfig.awsSes.awsRegion, credentials });
|
||||
|
||||
type SendEmailParams = {
|
||||
body: string;
|
||||
subject: string;
|
||||
recipients: string[];
|
||||
};
|
||||
|
||||
export async function sendEmail({ body, subject, recipients }: SendEmailParams) {
|
||||
const request: SendEmailRequest = {
|
||||
Destination: { ToAddresses: recipients },
|
||||
Message: {
|
||||
Body: {
|
||||
Text: {
|
||||
Charset: "UTF-8",
|
||||
Data: body,
|
||||
},
|
||||
},
|
||||
Subject: {
|
||||
Charset: "UTF-8",
|
||||
Data: subject,
|
||||
},
|
||||
},
|
||||
Source: serverRuntimeConfig.awsSes.fromEmail,
|
||||
};
|
||||
|
||||
await ses.sendEmail(request).promise();
|
||||
}
|
||||
83
package-lock.json
generated
83
package-lock.json
generated
@ -5449,6 +5449,68 @@
|
||||
"resolved": "https://registry.npmjs.org/awesome-phonenumber/-/awesome-phonenumber-2.58.0.tgz",
|
||||
"integrity": "sha512-rsbIn7Htq/QqUfJ7E53oGiGnLca5SUJEshg8zG5h9WK+fTxoGA12/NDKC5eCvkK2eaP8gR/RVA1yuf0Arib7vg=="
|
||||
},
|
||||
"aws-sdk": {
|
||||
"version": "2.985.0",
|
||||
"resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.985.0.tgz",
|
||||
"integrity": "sha512-Al1oFENrrDeKRpxlklk5sONqzCgEkrhaJ1vtIfpLYYqhNlAY+ku/z1hG1+qSlvgmljGyn7T6/zAb2EcbbAFZLQ==",
|
||||
"requires": {
|
||||
"buffer": "4.9.2",
|
||||
"events": "1.1.1",
|
||||
"ieee754": "1.1.13",
|
||||
"jmespath": "0.15.0",
|
||||
"querystring": "0.2.0",
|
||||
"sax": "1.2.1",
|
||||
"url": "0.10.3",
|
||||
"uuid": "3.3.2",
|
||||
"xml2js": "0.4.19"
|
||||
},
|
||||
"dependencies": {
|
||||
"buffer": {
|
||||
"version": "4.9.2",
|
||||
"resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz",
|
||||
"integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==",
|
||||
"requires": {
|
||||
"base64-js": "^1.0.2",
|
||||
"ieee754": "^1.1.4",
|
||||
"isarray": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"events": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz",
|
||||
"integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ="
|
||||
},
|
||||
"ieee754": {
|
||||
"version": "1.1.13",
|
||||
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
|
||||
"integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg=="
|
||||
},
|
||||
"punycode": {
|
||||
"version": "1.3.2",
|
||||
"resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz",
|
||||
"integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0="
|
||||
},
|
||||
"querystring": {
|
||||
"version": "0.2.0",
|
||||
"resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
|
||||
"integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA="
|
||||
},
|
||||
"url": {
|
||||
"version": "0.10.3",
|
||||
"resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz",
|
||||
"integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=",
|
||||
"requires": {
|
||||
"punycode": "1.3.2",
|
||||
"querystring": "0.2.0"
|
||||
}
|
||||
},
|
||||
"uuid": {
|
||||
"version": "3.3.2",
|
||||
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
|
||||
"integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"axe-core": {
|
||||
"version": "4.3.3",
|
||||
"resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.3.3.tgz",
|
||||
@ -17611,6 +17673,11 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"sax": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz",
|
||||
"integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o="
|
||||
},
|
||||
"saxes": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz",
|
||||
@ -20378,6 +20445,22 @@
|
||||
"resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz",
|
||||
"integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw=="
|
||||
},
|
||||
"xml2js": {
|
||||
"version": "0.4.19",
|
||||
"resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz",
|
||||
"integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==",
|
||||
"requires": {
|
||||
"sax": ">=0.6.0",
|
||||
"xmlbuilder": "~9.0.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"xmlbuilder": {
|
||||
"version": "9.0.7",
|
||||
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz",
|
||||
"integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0="
|
||||
}
|
||||
}
|
||||
},
|
||||
"xmlbuilder": {
|
||||
"version": "13.0.2",
|
||||
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-13.0.2.tgz",
|
||||
|
||||
@ -48,6 +48,7 @@
|
||||
"@tailwindcss/typography": "0.4.1",
|
||||
"@twilio/voice-sdk": "2.0.1",
|
||||
"awesome-phonenumber": "2.58.0",
|
||||
"aws-sdk": "2.985.0",
|
||||
"blitz": "0.40.0-canary.7",
|
||||
"clsx": "1.1.1",
|
||||
"got": "11.8.2",
|
||||
|
||||
Loading…
Reference in New Issue
Block a user