From 52f73745b685c7f410090e7d6014e34aaf10a471 Mon Sep 17 00:00:00 2001 From: Dessa Simpson Date: Tue, 10 Nov 2020 16:51:26 -0700 Subject: [PATCH 01/42] Add prod dockerfile --- Dockerfile.prod | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 Dockerfile.prod diff --git a/Dockerfile.prod b/Dockerfile.prod new file mode 100644 index 0000000..d0f3257 --- /dev/null +++ b/Dockerfile.prod @@ -0,0 +1,9 @@ +FROM node:latest + +COPY . /home/app +WORKDIR /home/app +USER node +ENV PORT 3000 +EXPOSE 3000 + +CMD ["ts-node", "./src/app.ts"] From 88955ba7074c0ce81a6c0620f1f97564fa2012a9 Mon Sep 17 00:00:00 2001 From: Dessa Simpson Date: Sat, 28 Nov 2020 22:05:54 -0700 Subject: [PATCH 02/42] Shorten container names in docker-compose.yml --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index aa08979..818df75 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3.8' services: app: - container_name: learn-request-queue + container_name: lrq build: . depends_on: - db @@ -11,7 +11,7 @@ services: - "80:3000" env_file: .env db: - container_name: learn-request-queue-db + container_name: lrqdb image: postgres ports: - "5432:5432" From 42d2a4a78cb58a2bb8bbf24d890e639f54e90157 Mon Sep 17 00:00:00 2001 From: Dessa Simpson Date: Sat, 28 Nov 2020 22:09:06 -0700 Subject: [PATCH 03/42] Add ISO date to logging --- src/logging.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/logging.ts b/src/logging.ts index 8ed1376..46db51d 100644 --- a/src/logging.ts +++ b/src/logging.ts @@ -10,5 +10,5 @@ export enum LogLevel { export async function log(logLevel: LogLevel, logMessage: any) { if (config.logLevel >= logLevel) - console.log(LogLevel[logLevel].padStart(7) + ' | ' + logMessage) + console.log(new Date().toISOString() + LogLevel[logLevel].padStart(7) + ' | ' + logMessage) } From a86e8a5667d481d90885dea6ff6009c394fadcb4 Mon Sep 17 00:00:00 2001 From: Dessa Simpson Date: Sat, 28 Nov 2020 22:05:08 -0700 Subject: [PATCH 04/42] Implement cron Fixes #22 --- db/10-cron.sql | 11 ++++++ src/app.ts | 59 +++------------------------- src/config.ts | 9 ++++- src/cron.ts | 52 ++++++++++++++++++++++++ src/cronjobs/index.ts | 7 ++++ src/cronjobs/processBans.ts | 40 +++++++++++++++++++ src/cronjobs/processEmptyMetadata.ts | 12 ++++++ src/queries.ts | 13 ++++++ src/twitch.ts | 21 ++++++---- 9 files changed, 161 insertions(+), 63 deletions(-) create mode 100644 db/10-cron.sql create mode 100644 src/cron.ts create mode 100644 src/cronjobs/index.ts create mode 100644 src/cronjobs/processBans.ts create mode 100644 src/cronjobs/processEmptyMetadata.ts diff --git a/db/10-cron.sql b/db/10-cron.sql new file mode 100644 index 0000000..3493563 --- /dev/null +++ b/db/10-cron.sql @@ -0,0 +1,11 @@ +CREATE TABLE cron ( + jobName varchar UNIQUE NOT NULL, -- Application-recognizable name for the job + runinterval interval NOT NULL, -- Duration between runs + -- Last successful run - only gets updated if run is successful + lastSuccess timestamptz DEFAULT to_timestamp(0), -- Defaults to beginning of time + PRIMARY KEY(jobName) +); + +INSERT INTO cron (jobName,runInterval) VALUES +('processBans','30 minutes'), +('processEmptyMetadata','1 day'); diff --git a/src/app.ts b/src/app.ts index e67ab26..dc50ec8 100644 --- a/src/app.ts +++ b/src/app.ts @@ -2,13 +2,13 @@ import * as config from "./config"; import * as requests from "./requests"; import * as twitch from "./twitch"; import * as queries from "./queries"; -import { log, LogLevel } from "./logging" import { URLSearchParams } from "url"; import express from "express"; import session from "express-session"; import pg from "pg"; import pgSessionStore from "connect-pg-simple"; import fetch, { Response as FetchResponse } from "node-fetch"; +import cron from "./cron"; import db from "./db"; import errorHandler from "./errors"; import * as version from "./version"; @@ -332,61 +332,12 @@ app.get("/", async (request, response) => { //}); // Logout -app.get("/logout", async (request, response) => request.session!.destroy(() => response.redirect(307, '/'))); - -async function processBannedUsers() { - log(LogLevel.INFO,"processBannedUsers run at " + new Date().toISOString()); - var streamer = await db.query(queries.getStreamerIdToken).then((result: pg.QueryResult) => result.rows[0]); - if (typeof streamer == 'undefined') return; - var response = await twitch.streamerApiRequest("/moderation/banned?broadcaster_id=" + streamer['userid']); - var dbconn = await db.connect(); - try { - await dbconn.query('BEGIN'); - dbconn.query("DELETE FROM bans"); - log(LogLevel.DEBUG,"Ban list:") - log(LogLevel.DEBUG,JSON.stringify(response.data,null,2)); - var insertBanQuery = "INSERT INTO bans (userid) VALUES "; - var banRow = 0; - var bansArray: number[] = []; - for (var ban of response.data) { - if (ban.expires_at == '') { - banRow++; - insertBanQuery += `($${banRow}), `; - bansArray.push(ban.user_id as number); - } - } - var banQueryConfig = { - text: insertBanQuery.slice(0,-2), // Cut last `, ` off of the end - values: bansArray - }; - log(LogLevel.DEBUG,"banQueryConfig object:") - log(LogLevel.DEBUG,JSON.stringify(banQueryConfig,null,2)) - dbconn.query(banQueryConfig); - dbconn.query("CALL update_scores()"); - await dbconn.query('COMMIT'); - } catch (e) { - await dbconn.query('ROLLBACK'); - } finally { - dbconn.release(); - } - setTimeout(processBannedUsers,3600000+Math.floor(Math.random()*900000)) // Run every 1-1.25 hours to balance load -} - -setTimeout(processBannedUsers,600000+Math.floor(Math.random()*600000)) - -async function processEmptyMetadata() { - log(LogLevel.INFO,"processEmptyMetadata run at " + new Date().toISOString()); - var result = await db.query(queries.getRequestsWithEmptyMetadata); - for (var row of result.rows) { - log(LogLevel.DEBUG,"Processing empty metadata for request: " + row['url']); - requests.updateRequestMetadata(row['url']); - } - setTimeout(processEmptyMetadata,3600000+Math.floor(Math.random()*900000)) // Run every 1-1.25 hours to balance load -} - -processEmptyMetadata(); +app.get("/logout", async (request, response) => + request.session!.destroy(() => response.redirect(307, '/'))); // Check version then listen version.checkVersion().then(_ => app.listen(config.port, () => { + cron.run(); + setInterval(cron.run,config.cronInterval); console.log(`Listening on port ${config.port}`); })); diff --git a/src/config.ts b/src/config.ts index ce3e768..3d9c47e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -36,6 +36,13 @@ if (typeof process.env.YOUTUBE_SECRET == 'undefined') { } export const youtubeSecret: string = process.env.YOUTUBE_SECRET!; -export const logLevel = (process.env.LOG_LEVEL ? LogLevel[process.env.LOG_LEVEL as keyof typeof LogLevel] : LogLevel.ERROR ); +export const logLevel = (process.env.LOG_LEVEL + ? LogLevel[process.env.LOG_LEVEL as keyof typeof LogLevel] + : LogLevel.ERROR ); + +export const cronInterval: number = (process.env.CRON_INTERVAL + ? parseInt(process.env.CRON_INTERVAL as string, 10) + : 60000); + console.log("Running with logLevel = " + logLevel) diff --git a/src/cron.ts b/src/cron.ts new file mode 100644 index 0000000..256835e --- /dev/null +++ b/src/cron.ts @@ -0,0 +1,52 @@ +import cronjobs from "./cronjobs" +import * as queries from "./queries"; +import * as twitch from "./twitch"; +import { log, LogLevel } from "./logging" +import db from "./db"; +import pg from "pg"; + +interface CronJob { + (streamer: twitch.StreamerUserIdTokenPair): Promise +} + +async function run() { + // If instance is not yet set up, end processing at this point + var streamer = await db.query(queries.getStreamerIdToken).then( + (result: pg.QueryResult) => result.rows[0]); + if (typeof streamer == 'undefined') { + log(LogLevel.INFO,"Cron run skipped due to instance not being set up") + return; + } + log(LogLevel.INFO,"Begin cron run") + for (var job in cronjobs) { + log(LogLevel.INFO,"cron: Checking job " + job); + var dbconn = await db.connect(); + await dbconn.query("BEGIN"); + try { + var needsRun = (await dbconn.query( + Object.assign(queries.getAndLockCronJob,{values: [job]}) + )).rowCount; + if (needsRun) { + log(LogLevel.INFO,`cron: Database says job ${job} needs run; executing`); + await (cronjobs as { [key: string]: CronJob })[job](streamer) + log(LogLevel.INFO,`cron: Job ${job} returned without exception; updating lastSuccess`); + await dbconn.query(Object.assign(queries.updateCronJobLastSuccess,{values: [job]})) + await dbconn.query("COMMIT"); + } else { + log(LogLevel.INFO,`cron: Database says job ${job} does not need run; skipping`); + await dbconn.query("ROLLBACK"); + } + } catch(e) { + log(LogLevel.ERROR,`cron: Job ${job} threw exception; rolling back`); + await dbconn.query("ROLLBACK"); + log(LogLevel.ERROR,`cron: Job ${job} exception message: ${e}`) + } finally { + log(LogLevel.DEBUG,`cron: Job ${job} hit finally; releasing dbconn`); + await dbconn.release(); + } + log(LogLevel.INFO,"cron: Finished job " + job); + } + log(LogLevel.INFO,"End cron run") +} + +export = {run} diff --git a/src/cronjobs/index.ts b/src/cronjobs/index.ts new file mode 100644 index 0000000..0c856fd --- /dev/null +++ b/src/cronjobs/index.ts @@ -0,0 +1,7 @@ +import { processBans } from './processBans'; +import { processEmptyMetadata } from './processEmptyMetadata'; + +export = { + processBans, + processEmptyMetadata +} diff --git a/src/cronjobs/processBans.ts b/src/cronjobs/processBans.ts new file mode 100644 index 0000000..1fe84ab --- /dev/null +++ b/src/cronjobs/processBans.ts @@ -0,0 +1,40 @@ +import * as twitch from "../twitch"; +import { log, LogLevel } from "../logging" +import db from "../db"; + +export async function processBans(streamer: twitch.StreamerUserIdTokenPair) { + var response = await twitch.streamerApiRequest(streamer, + "/moderation/banned?broadcaster_id=" + streamer.userid); + var dbconn = await db.connect(); + try { + await dbconn.query('BEGIN'); + await dbconn.query("DELETE FROM bans"); + log(LogLevel.DEBUG,"Ban list:") + log(LogLevel.DEBUG,JSON.stringify(response.data,null,2)); + var insertBanQuery = "INSERT INTO bans (userid) VALUES "; + var banRow = 0; + var bansArray: number[] = []; + for (var ban of response.data) { + if (ban.expires_at == '') { + banRow++; + insertBanQuery += `($${banRow}), `; + bansArray.push(ban.user_id as number); + } + } + var banQueryConfig = { + text: insertBanQuery.slice(0,-2), // Cut last `, ` off of the end + values: bansArray + }; + log(LogLevel.DEBUG,"banQueryConfig object:") + log(LogLevel.DEBUG,JSON.stringify(banQueryConfig,null,2)) + await dbconn.query(banQueryConfig); + await dbconn.query("CALL update_scores()"); + await dbconn.query('COMMIT'); + } catch (e) { + log(LogLevel.ERROR,"cronjobs.processBans: Exception thrown; rolling back"); + await dbconn.query('ROLLBACK'); + throw(e); + } finally { + await dbconn.release(); + } +} diff --git a/src/cronjobs/processEmptyMetadata.ts b/src/cronjobs/processEmptyMetadata.ts new file mode 100644 index 0000000..9b911bb --- /dev/null +++ b/src/cronjobs/processEmptyMetadata.ts @@ -0,0 +1,12 @@ +import * as requests from "../requests"; +import * as queries from "../queries"; +import { log, LogLevel } from "../logging" +import db from "../db"; + +export async function processEmptyMetadata(_: any) { // TODO: Consume streamer userid for WHERE clause + var result = await db.query(queries.getRequestsWithEmptyMetadata); + for (var row of result.rows) { + log(LogLevel.DEBUG,"Processing empty metadata for request: " + row['url']); + requests.updateRequestMetadata(row['url']); + } +} diff --git a/src/queries.ts b/src/queries.ts index 4f1d3a5..7b6725e 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -111,3 +111,16 @@ export const getDbVersion = { name: "getDbVersion", text: "SELECT get_version()" } + +export const getAndLockCronJob = { + name: "getCronJob", + text: `SELECT * FROM cron + WHERE (lastSuccess + runInterval) < now() + AND jobName = $1 + FOR UPDATE SKIP LOCKED;` +} + +export const updateCronJobLastSuccess = { + name: "updateCronJobLastSuccess", + text: "UPDATE cron SET lastSuccess = now() WHERE jobName = $1" +} diff --git a/src/twitch.ts b/src/twitch.ts index e44b303..814ba1f 100644 --- a/src/twitch.ts +++ b/src/twitch.ts @@ -10,6 +10,11 @@ export interface TokenPair { refresh_token: string; } +export interface StreamerUserIdTokenPair { + userid: number + tokenpair: TokenPair +} + // Refresh the API token. Returns true on success and false on failure. async function refreshApiToken(tokens: TokenPair): Promise { log(LogLevel.DEBUG,`Call: twitch.refreshApiToken(${JSON.stringify(tokens,null,2)})`); @@ -85,15 +90,15 @@ export async function apiRequest(tokens: TokenPair, endpoint: string): Promise < }) } -export async function streamerApiRequest(endpoint: string) { +export async function streamerApiRequest(streamer: StreamerUserIdTokenPair, endpoint: string) { log(LogLevel.DEBUG,`Call: twitch.streamerApiRequest(${endpoint})`); - var streamer = await db.query(queries.getStreamerIdToken).then((result: pg.QueryResult) => result.rows[0]); - var tokenpair = streamer.tokenpair; - var originaltoken = tokenpair.access_token; - log(LogLevel.DEBUG,"Original token: " + tokenpair.access_token); - var response = await apiRequest(tokenpair,endpoint); - log(LogLevel.DEBUG,"New token: " + tokenpair.access_token); - if (tokenpair.access_token != originaltoken) await db.query(Object.assign(queries.updateStreamer,{ values: [streamer.userid,tokenpair] })) + var originaltoken = streamer.tokenpair.access_token; + log(LogLevel.DEBUG,"Original token: " + originaltoken); + var response = await apiRequest(streamer.tokenpair,endpoint); + log(LogLevel.DEBUG,"New token: " + streamer.tokenpair.access_token); + if (streamer.tokenpair.access_token != originaltoken) + await db.query(Object.assign(queries.updateStreamer, + { values: [streamer.userid,streamer.tokenpair] })) return response; } From af97647033f498b14011a65b69b4a6a9cf9264e3 Mon Sep 17 00:00:00 2001 From: Dessa Simpson Date: Sun, 29 Nov 2020 01:17:58 -0700 Subject: [PATCH 05/42] Make processBans support pagination --- src/cronjobs/processBans.ts | 58 ++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/src/cronjobs/processBans.ts b/src/cronjobs/processBans.ts index 1fe84ab..d21cb76 100644 --- a/src/cronjobs/processBans.ts +++ b/src/cronjobs/processBans.ts @@ -3,31 +3,49 @@ import { log, LogLevel } from "../logging" import db from "../db"; export async function processBans(streamer: twitch.StreamerUserIdTokenPair) { - var response = await twitch.streamerApiRequest(streamer, - "/moderation/banned?broadcaster_id=" + streamer.userid); - var dbconn = await db.connect(); + var dbconn = await db.connect(); try { await dbconn.query('BEGIN'); await dbconn.query("DELETE FROM bans"); - log(LogLevel.DEBUG,"Ban list:") - log(LogLevel.DEBUG,JSON.stringify(response.data,null,2)); - var insertBanQuery = "INSERT INTO bans (userid) VALUES "; - var banRow = 0; - var bansArray: number[] = []; - for (var ban of response.data) { - if (ban.expires_at == '') { - banRow++; - insertBanQuery += `($${banRow}), `; - bansArray.push(ban.user_id as number); + //log(LogLevel.DEBUG,"Ban API response:") + //log(LogLevel.DEBUG,JSON.stringify(response,null,2)); + var response = await twitch.streamerApiRequest(streamer, + `/moderation/banned?broadcaster_id=${streamer.userid}&first=100`); + while (true) { + var insertBanQuery = "INSERT INTO bans (userid) VALUES "; + var banRow = 0; + var bansArray: number[] = []; + if (Object.keys(response.data).length > 0) { + for (var ban of response.data) { + if (ban.expires_at == '') { + banRow++; + insertBanQuery += `($${banRow}), `; + bansArray.push(ban.user_id as number); + } + } + insertBanQuery = insertBanQuery.slice(0,-2); // Cut last `, ` off of the end + insertBanQuery += " ON CONFLICT DO NOTHING"; // Deal with broken endpoint returning dupes + var banQueryConfig = { + text: insertBanQuery, + values: bansArray + }; + log(LogLevel.DEBUG,"banQueryConfig object:") + log(LogLevel.DEBUG,JSON.stringify(banQueryConfig,null,2)) + await dbconn.query(banQueryConfig); + } + if (response.pagination.cursor) { + var oldFirstUserid = response.data[0].user_id; + response = await twitch.streamerApiRequest(streamer, + `/moderation/banned?broadcaster_id=${streamer.userid}&after=${response.pagination.cursor}&first=100`); + // Work around broken api endpoint giving a cursor referring to the + // current page, causing an infinite loop + if (oldFirstUserid == response.data[0].user_id) break; + //log(LogLevel.DEBUG,"Ban API response:"); + //log(LogLevel.DEBUG,JSON.stringify(response,null,2)); + } else { + break; } } - var banQueryConfig = { - text: insertBanQuery.slice(0,-2), // Cut last `, ` off of the end - values: bansArray - }; - log(LogLevel.DEBUG,"banQueryConfig object:") - log(LogLevel.DEBUG,JSON.stringify(banQueryConfig,null,2)) - await dbconn.query(banQueryConfig); await dbconn.query("CALL update_scores()"); await dbconn.query('COMMIT'); } catch (e) { From aded3f4c770f2477170d9c111bbb0ef254ec7879 Mon Sep 17 00:00:00 2001 From: Dessa Simpson Date: Sun, 29 Nov 2020 01:24:41 -0700 Subject: [PATCH 06/42] Remove extraneous whitespace --- src/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.ts b/src/config.ts index 3d9c47e..a7ab483 100644 --- a/src/config.ts +++ b/src/config.ts @@ -39,7 +39,7 @@ export const youtubeSecret: string = process.env.YOUTUBE_SECRET!; export const logLevel = (process.env.LOG_LEVEL ? LogLevel[process.env.LOG_LEVEL as keyof typeof LogLevel] : LogLevel.ERROR ); - + export const cronInterval: number = (process.env.CRON_INTERVAL ? parseInt(process.env.CRON_INTERVAL as string, 10) : 60000); From 2751ccf846bb18883345dace571c56757354c670 Mon Sep 17 00:00:00 2001 From: Dessa Simpson Date: Sun, 29 Nov 2020 12:37:30 -0700 Subject: [PATCH 07/42] Update Dockerfile.prod and .dockerignore --- .dockerignore | 3 +++ Dockerfile.prod | 27 +++++++++++++++++++++------ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/.dockerignore b/.dockerignore index d95cec9..fe9214f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,7 @@ Dockerfile +Dockerfile.prod +docker-compose.yml .dockerignore .git .gitignore +node_modules diff --git a/Dockerfile.prod b/Dockerfile.prod index d0f3257..49539fd 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -1,9 +1,24 @@ -FROM node:latest +FROM node:14 AS builder -COPY . /home/app -WORKDIR /home/app USER node -ENV PORT 3000 -EXPOSE 3000 -CMD ["ts-node", "./src/app.ts"] +ENV PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/app/node_modules/.bin" + +COPY --chown=node . /app +WORKDIR /app +RUN ["npm", "install"] +RUN ["tsc", "--outDir", "build"] + +FROM node:14-alpine + +EXPOSE 3000 +ENV PORT 3000 +ENV PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/app/node_modules/.bin" NODE_ENV="production" + +USER node +WORKDIR /app +COPY --from=builder /app/build build +COPY --from=builder /app/node_modules node_modules +COPY --from=builder /app/public public +COPY --from=builder /app/views views +CMD ["node", "./build/app.js"] From 35aa0914a2f8366d052d2cea33cc5f6f1c7b9a99 Mon Sep 17 00:00:00 2001 From: Dessa Simpson Date: Sun, 29 Nov 2020 13:29:10 -0700 Subject: [PATCH 08/42] Remove unnecessary import from twitch.ts --- src/twitch.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/twitch.ts b/src/twitch.ts index 814ba1f..369e307 100644 --- a/src/twitch.ts +++ b/src/twitch.ts @@ -2,7 +2,6 @@ import * as config from "./config"; import * as queries from "./queries"; import { log, LogLevel } from "./logging" import fetch, { Response as FetchResponse } from "node-fetch"; -import pg from "pg"; import db from "./db"; export interface TokenPair { From 21650994d2aaf713fdca9bcb0d8e21fbb17f133e Mon Sep 17 00:00:00 2001 From: Dessa Simpson Date: Sun, 29 Nov 2020 13:30:27 -0700 Subject: [PATCH 09/42] Implement checkVersion middleware Checks version on each request, terminating the application if version mdoes not match --- src/app.ts | 8 +++++++- src/version.ts | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/app.ts b/src/app.ts index dc50ec8..494aae9 100644 --- a/src/app.ts +++ b/src/app.ts @@ -8,6 +8,7 @@ import session from "express-session"; import pg from "pg"; import pgSessionStore from "connect-pg-simple"; import fetch, { Response as FetchResponse } from "node-fetch"; +import { log, LogLevel } from "./logging" import cron from "./cron"; import db from "./db"; import errorHandler from "./errors"; @@ -26,6 +27,7 @@ async function validateApiToken(session: Express.Session) { } const app = express(); +app.use(version.checkVersionMiddleware); app.use(express.static('public')); app.use(express.urlencoded({extended: false})); app.use(session({ @@ -340,4 +342,8 @@ version.checkVersion().then(_ => app.listen(config.port, () => { cron.run(); setInterval(cron.run,config.cronInterval); console.log(`Listening on port ${config.port}`); -})); +})) +.catch((e) => { + log(LogLevel.ERROR,e) + process.exit(1); +}); diff --git a/src/version.ts b/src/version.ts index cfe9591..434c3bd 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,4 +1,6 @@ import * as queries from "./queries"; +import { log, LogLevel } from "./logging" +import express from "express"; import pg from "pg"; import db from "./db"; @@ -10,6 +12,19 @@ export function getVersion() { return `${versionMajor}.${versionMinor}.${versionPatch}` } +export async function checkVersionMiddleware(_req: express.Request, _res: express.Response, next: express.NextFunction) { + try { + await checkVersion(); + } catch (e) { + log(LogLevel.ERROR,e) + // Terminate the nodejs process with error. If restarted, the app will + // never start listening for new requests so this will not result in a + // restart loop. + process.exit(1); + } + next(); +} + export async function checkVersion() { var dbver = await db.query(queries.getDbVersion).then((result: pg.QueryResult) => result.rows[0]['get_version']); if (dbver != `${versionMajor}.${versionMinor}`) { From f69e3a821c61ee49da06ae7d10502aa331f06123 Mon Sep 17 00:00:00 2001 From: Dessa Simpson Date: Sun, 29 Nov 2020 13:31:03 -0700 Subject: [PATCH 10/42] Add v0.2-v0.3 db upgrade script --- db/upgrade/v0.2-v0.3.sql | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 db/upgrade/v0.2-v0.3.sql diff --git a/db/upgrade/v0.2-v0.3.sql b/db/upgrade/v0.2-v0.3.sql new file mode 100644 index 0000000..defe309 --- /dev/null +++ b/db/upgrade/v0.2-v0.3.sql @@ -0,0 +1,18 @@ +BEGIN; + +UPDATE version SET minor = 3; + +CREATE TABLE cron ( + jobName varchar UNIQUE NOT NULL, -- Application-recognizable name for the job + runinterval interval NOT NULL, -- Duration between runs + -- Last successful run - only gets updated if run is successful + lastSuccess timestamptz DEFAULT to_timestamp(0), -- Defaults to beginning of time + PRIMARY KEY(jobName) +); + +INSERT INTO cron (jobName,runInterval) VALUES +('processBans','30 minutes'), +('processEmptyMetadata','1 day'); + + +COMMIT; From 6d559b5fe256631a2f9d87d33343cf523f666eec Mon Sep 17 00:00:00 2001 From: Dessa Simpson Date: Sun, 29 Nov 2020 13:36:30 -0700 Subject: [PATCH 11/42] Bump version to v0.3 --- src/version.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/version.ts b/src/version.ts index 434c3bd..18b3f83 100644 --- a/src/version.ts +++ b/src/version.ts @@ -5,8 +5,8 @@ import pg from "pg"; import db from "./db"; var versionMajor = 0; -var versionMinor = 2; -var versionPatch = 1; +var versionMinor = 3; +var versionPatch = 0; export function getVersion() { return `${versionMajor}.${versionMinor}.${versionPatch}` From 57991dc36acf6dba3393ffd5e109e57010519488 Mon Sep 17 00:00:00 2001 From: Dessa Simpson Date: Sun, 29 Nov 2020 14:13:26 -0700 Subject: [PATCH 12/42] Stringify twitchApiToken response for debug logging --- src/twitch.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/twitch.ts b/src/twitch.ts index 369e307..0569bee 100644 --- a/src/twitch.ts +++ b/src/twitch.ts @@ -29,8 +29,8 @@ async function refreshApiToken(tokens: TokenPair): Promise { if (res.status == 200) { log(LogLevel.INFO,"twitch.refreshApiToken: Refresh returned success."); var data = await (res.json() as Promise); - log(LogLevel.DEBUG, "Returned data:") - log(LogLevel.DEBUG, data) + log(LogLevel.DEBUG, "Returned data:"); + log(LogLevel.DEBUG, JSON.stringify(data,null,2)); tokens.access_token = data.access_token; tokens.refresh_token = data.refresh_token; return true; From 7a4c39353c6d1f0c996f6f1358f0205b8363518b Mon Sep 17 00:00:00 2001 From: Dessa Simpson Date: Sun, 20 Dec 2020 10:43:01 -0700 Subject: [PATCH 13/42] Bump version in version.sql (forgot to in e83dbe51) --- db/00-version.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/00-version.sql b/db/00-version.sql index 5c43aec..938accc 100644 --- a/db/00-version.sql +++ b/db/00-version.sql @@ -8,4 +8,4 @@ CREATE OR REPLACE FUNCTION get_version() RETURNS VARCHAR AS $$SELECT major || '.' || minor FROM version $$ LANGUAGE SQL; -INSERT INTO version (major,minor) VALUES (0,2); +INSERT INTO version (major,minor) VALUES (0,3); From 2e5762c0296a0f62f87712f0821def24d70191a8 Mon Sep 17 00:00:00 2001 From: Dessa Simpson Date: Sun, 20 Dec 2020 13:16:15 -0700 Subject: [PATCH 14/42] Add release checklist (RELEASE.md) --- RELEASE.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 RELEASE.md diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..f1c186f --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,20 @@ +# Release Checklist + +## Release Type +- Were there any database schema changes? +- Are there any significant UI changes? +- Have any significant new features been added? +- Are there any API changes? + +If the answer to any of the above is yes, then the release MUST be a major or minor release. Otherwise, the release MAY be a patch release (at developer's discretion). + +## Major/Minor Releases +- [ ] Add a commit which adds a database upgrade script at `db/upgrade/[oldversion]-[newversion].sql` + - ALWAYS use a transaction for the entirety of this file + - At minimum, DB version must be bumped +- [ ] Add a commit which bumps the version in `app/version.ts` and `db/00-version.sql` +- [ ] Tag the latest commit with `vMAJOR.MINOR` + +## Patch Releases +- [ ] Add a commit which bumps the patch level in `app/version.ts` +- [ ] Tag the latest commit with `vMAJOR.MINOR.PATCH` From 1c34b3f0139fcad8ecb520ceb1aea2f731bbc7b7 Mon Sep 17 00:00:00 2001 From: Dessa Simpson Date: Sun, 21 Feb 2021 20:13:47 -0700 Subject: [PATCH 15/42] Get page title and colors from database See #18 --- db/05-config.sql | 6 ++++-- src/app.ts | 14 +++++++++++++- src/queries.ts | 5 +++++ views/colors.eta | 41 +++++++++++++++++++++++++++++++++++++++++ views/main.eta | 7 ++++--- 5 files changed, 67 insertions(+), 6 deletions(-) create mode 100644 views/colors.eta diff --git a/db/05-config.sql b/db/05-config.sql index edf6d9e..1c4b65a 100644 --- a/db/05-config.sql +++ b/db/05-config.sql @@ -3,8 +3,10 @@ CREATE TABLE config ( normaluservotepoints int NOT NULL, followervotepoints int NOT NULL, subscribervotepoints int NOT NULL, + title varchar NOT NULL, + colors jsonb NOT NULL, PRIMARY KEY (rowLock) ); -INSERT INTO config (normalUserVotePoints,followerVotePoints,subscriberVotePoints) - VALUES (10,50,100); +INSERT INTO config (normalUserVotePoints,followerVotePoints,subscriberVotePoints,title,colors) + VALUES (10,50,100,'{username}''s Learn Request Queue','{"bg": {"primary": "#444444","table": "#282828","navbar": "#666666","error": "#ff0000"},"fg": {"primary": "#dddddd","ahover": "#ffffff","title": "#eeeeee"}}'); diff --git a/src/app.ts b/src/app.ts index 494aae9..3bdc436 100644 --- a/src/app.ts +++ b/src/app.ts @@ -302,7 +302,7 @@ app.get("/callback", async (request, response) => { app.get("/", async (request, response) => { if (request.session) await validateApiToken(request.session); var streamerInfo = await db.query(queries.getStreamerInfo).then((result: pg.QueryResult) => result.rows[0]); - var validStates = JSON.stringify((await db.query(queries.getValidStates).then((result: pg.QueryResult) => result.rows)).map((row: any) => row.state)); + var config = await db.query(queries.getConfig).then((result: pg.QueryResult) => result.rows[0]); if (typeof streamerInfo == 'undefined') { response.redirect(307, `https://id.twitch.tv/oauth2/authorize?client_id=${config.twitchClientId}&redirect_uri=${config.urlPrefix}/callback&response_type=code&scope=channel:read:subscriptions moderation:read`); return; @@ -312,22 +312,34 @@ app.get("/", async (request, response) => { loggedIn: false, clientId: config.twitchClientId, urlPrefix: config.urlPrefix, + pageTitle: config.title, streamerName: streamerInfo['displayname'], streamerProfilePicture: streamerInfo['imageurl'] }); } else { + var validStates = JSON.stringify((await db.query(queries.getValidStates).then((result: pg.QueryResult) => result.rows)).map((row: any) => row.state)); response.render('main.eta', { loggedIn: true, userName: request.session.user.display_name, userProfilePicture: request.session.user.profile_image_url, validStates: validStates, isStreamer: streamerInfo['userid'] == request.session.user.id, + pageTitle: config.title, streamerName: streamerInfo['displayname'], streamerProfilePicture: streamerInfo['imageurl'] }); } }); +app.get("/colors.css", async (_request, response) => { + var streamerInfo = await db.query(queries.getStreamerInfo).then((result: pg.QueryResult) => result.rows[0]); + var colors = await db.query(queries.getConfig).then((result: pg.QueryResult) => result.rows[0]['colors']); + console.log(colors); + if (typeof streamerInfo == 'undefined') return; + response.contentType("text/css"); + response.render('colors.eta', colors); +}); + // Streamer Panel //app.get("/streamer/", async (request, response) => { // diff --git a/src/queries.ts b/src/queries.ts index 7b6725e..b9eeeb7 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -26,6 +26,11 @@ export const getStreamerInfo = { text: "SELECT userid,displayname,imageurl FROM streamer_user_vw" } +export const getConfig = { + name: "getConfig", + text: "SELECT * FROM config" +} + export const updateStreamer = { name: "updateStreamer", text: "INSERT INTO streamer (userid,tokenPair) VALUES ($1,$2)\ diff --git a/views/colors.eta b/views/colors.eta new file mode 100644 index 0000000..b98c5da --- /dev/null +++ b/views/colors.eta @@ -0,0 +1,41 @@ +body { + background-color: <%= it.bg.primary %>; + color: <%= it.fg.primary %>; +} + +a { + color: <%= it.fg.primary %>; +} + +a:hover { + color: <%= it.fg.ahover %>; +} + +.error { + background-color: <%= it.bg.error %>; +} + +#navbar { + background-color: <%= it.bg.navbar %>; +} + +#nav-title, #nav-title a { + color: <%= it.fg.title %>; +} + +#main { + background-color: <%= it.bg.table %>; +} + +#modalBackground { + background-color: <%= it.bg.primary %>; + background-color: <%= it.bg.primary %>aa; +} + +.modal { + background-color: <%= it.bg.primary %>; +} + +#deleteRequestLink { + color: <%= it.bg.error %>; +} diff --git a/views/main.eta b/views/main.eta index 90f9f82..c04023e 100644 --- a/views/main.eta +++ b/views/main.eta @@ -1,8 +1,9 @@ - - <%= it.streamerName %>'s Learn Request Queue + + + <%= it.pageTitle.replace('{username}',it.streamerName) %>