From 891ccfae92ca3e1ec776874edab7da108ffa398f Mon Sep 17 00:00:00 2001 From: Dessa Simpson Date: Mon, 26 Apr 2021 05:16:43 +0000 Subject: [PATCH 1/6] Update 'RELEASE.md' --- RELEASE.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index f1c186f..c60d7c9 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -12,9 +12,19 @@ If the answer to any of the above is yes, then the release MUST be a major or mi - [ ] Add a commit which adds a database upgrade script at `db/upgrade/[oldversion]-[newversion].sql` - ALWAYS use a transaction for the entirety of this file - At minimum, DB version must be bumped -- [ ] Add a commit which bumps the version in `app/version.ts` and `db/00-version.sql` + - Test the upgrade script as follows: + ``` + git checkout [previous release tag] + docker-compose down + docker-compose up + psql -h 0 -U postgres < db/upgrade/v[previous]-v[current].sql + ``` + Update the version in `src/version.ts` and verify the app works as expected. +- [ ] Add a commit which bumps the version in `src/version.ts` and `db/00-version.sql`, entitled `Bump version to vMAJOR.MINOR` - [ ] Tag the latest commit with `vMAJOR.MINOR` +- [ ] Write release notes ## Patch Releases -- [ ] Add a commit which bumps the patch level in `app/version.ts` +- [ ] Add a commit which bumps the patch level in `src/version.ts`, entitled `Bump version to vMAJOR.MINOR.PATCH` - [ ] Tag the latest commit with `vMAJOR.MINOR.PATCH` +- [ ] Write release notes \ No newline at end of file From 7897fe6f3dc2a1080a024c9a47689c68414e0f48 Mon Sep 17 00:00:00 2001 From: Dessa Simpson Date: Sun, 6 Jun 2021 23:11:30 -0700 Subject: [PATCH 2/6] Add ratelimits Fixes #10 --- db/05-config.sql | 16 +++++++++------- db/90-views.sql | 19 +++++++++++++++++++ db/upgrade/v0.7-v0.8.sql | 34 ++++++++++++++++++++++++++++++++++ src/queries.ts | 5 +++++ src/requests.ts | 22 ++++++++++++++++++---- 5 files changed, 85 insertions(+), 11 deletions(-) create mode 100644 db/upgrade/v0.7-v0.8.sql diff --git a/db/05-config.sql b/db/05-config.sql index 1c4b65a..2e830ab 100644 --- a/db/05-config.sql +++ b/db/05-config.sql @@ -1,12 +1,14 @@ CREATE TABLE config ( rowlock bool DEFAULT TRUE UNIQUE NOT NULL CHECK (rowlock = TRUE), - normaluservotepoints int NOT NULL, - followervotepoints int NOT NULL, - subscribervotepoints int NOT NULL, - title varchar NOT NULL, - colors jsonb NOT NULL, + normaluservotepoints int NOT NULL DEFAULT 10, + followervotepoints int NOT NULL DEFAULT 50, + subscribervotepoints int NOT NULL DEFAULT 100, + normaluserratelimit int NOT NULL DEFAULT 1, + followerratelimit int NOT NULL DEFAULT 2, + subscriberratelimit int NOT NULL DEFAULT 3, + title varchar NOT NULL DEFAULT '{username}''s Learn Request Queue', + colors jsonb NOT NULL DEFAULT '{"bg": {"primary": "#444444","table": "#282828","navbar": "#666666","error": "#ff0000"},"fg": {"primary": "#dddddd","ahover": "#ffffff","title": "#eeeeee"}}', PRIMARY KEY (rowLock) ); -INSERT INTO config (normalUserVotePoints,followerVotePoints,subscriberVotePoints,title,colors) - VALUES (10,50,100,'{username}''s Learn Request Queue','{"bg": {"primary": "#444444","table": "#282828","navbar": "#666666","error": "#ff0000"},"fg": {"primary": "#dddddd","ahover": "#ffffff","title": "#eeeeee"}}'); +INSERT INTO config (rowlock) VALUES (true); diff --git a/db/90-views.sql b/db/90-views.sql index 14896b0..a31ec16 100644 --- a/db/90-views.sql +++ b/db/90-views.sql @@ -60,3 +60,22 @@ CREATE OR REPLACE VIEW streamer_user_vw AS SELECT users.userid as userid, users.displayname as displayname, users.imageurl as imageurl FROM streamer LEFT JOIN users ON streamer.userid = users.userid; + +CREATE OR REPLACE VIEW ratelimit_vw AS + SELECT users.userid,COALESCE(count,0),ratelimit.reqcount AS max,COALESCE(count,0) >= ratelimit.reqcount AS status + FROM users + LEFT JOIN (SELECT requester,COUNT(url) + FROM requests + WHERE reqtimestamp > (now() - '24 hours'::interval) + GROUP BY requests.requester + ) AS requests ON users.userid = requests.requester + LEFT JOIN follows ON requests.requester = follows.userid + LEFT JOIN subscriptions ON requests.requester = subscriptions.userid + CROSS JOIN config + CROSS JOIN LATERAL (VALUES ( + CASE + WHEN follows.userid IS NULL AND subscriptions.userid IS NULL THEN config.normaluserratelimit + WHEN follows.userid IS NOT NULL AND subscriptions.userid IS NULL THEN config.followerratelimit + WHEN subscriptions.userid IS NOT NULL THEN config.subscriberratelimit + END + )) AS ratelimit(reqcount); diff --git a/db/upgrade/v0.7-v0.8.sql b/db/upgrade/v0.7-v0.8.sql new file mode 100644 index 0000000..c5d5ab7 --- /dev/null +++ b/db/upgrade/v0.7-v0.8.sql @@ -0,0 +1,34 @@ +BEGIN; + +ALTER TABLE config + ALTER COLUMN normaluservotepoints SET DEFAULT 10, + ALTER COLUMN followervotepoints SET DEFAULT 50, + ALTER COLUMN subscribervotepoints SET DEFAULT 100, + ALTER COLUMN title SET DEFAULT '{username}''s Learn Request Queue', + ALTER COLUMN colors SET DEFAULT '{"bg": {"primary": "#444444","table": "#282828","navbar": "#666666","error": "#ff0000"},"fg": {"primary": "#dddddd","ahover": "#ffffff","title": "#eeeeee"}}', + ADD COLUMN normaluserratelimit int NOT NULL DEFAULT 1, + ADD COLUMN followerratelimit int NOT NULL DEFAULT 2, + ADD COLUMN subscriberratelimit int NOT NULL DEFAULT 3; + +CREATE OR REPLACE VIEW ratelimit_vw AS + SELECT users.userid,COALESCE(count,0),ratelimit.reqcount AS max,COALESCE(count,0) >= ratelimit.reqcount AS status + FROM users + LEFT JOIN (SELECT requester,COUNT(url) + FROM requests + WHERE reqtimestamp > (now() - '24 hours'::interval) + GROUP BY requests.requester + ) AS requests ON users.userid = requests.requester + LEFT JOIN follows ON requests.requester = follows.userid + LEFT JOIN subscriptions ON requests.requester = subscriptions.userid + CROSS JOIN config + CROSS JOIN LATERAL (VALUES ( + CASE + WHEN follows.userid IS NULL AND subscriptions.userid IS NULL THEN config.normaluserratelimit + WHEN follows.userid IS NOT NULL AND subscriptions.userid IS NULL THEN config.followerratelimit + WHEN subscriptions.userid IS NOT NULL THEN config.subscriberratelimit + END + )) AS ratelimit(reqcount); + + +ROLLBACK; +--COMMIT; diff --git a/src/queries.ts b/src/queries.ts index 09c615b..8fa54e6 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -130,3 +130,8 @@ export const updateCronJobLastSuccess = { name: "updateCronJobLastSuccess", text: "UPDATE cron SET lastSuccess = now() WHERE jobName = $1" } + +export const getRateLimitStatus = { + name: "getRateLimitStatus", + text: "SELECT * FROM ratelimit_vw WHERE userid = $1" +} diff --git a/src/requests.ts b/src/requests.ts index 0033526..1a35094 100644 --- a/src/requests.ts +++ b/src/requests.ts @@ -87,6 +87,7 @@ async function retrieveYoutubeMetadata(url: string) { } export async function addRequest(url: string, requester: string): Promise<[number,string]> { + // Check that URL is of an accepted format var validUrl = false; for (var regex of validUrlRegexes) { if (regex.test(url)) { @@ -95,11 +96,24 @@ export async function addRequest(url: string, requester: string): Promise<[numbe } } if (!validUrl) return [400, "Invalid song URL."]; - var result = await checkRequestExists(url) - if (result) { - console.log(result); - return [200,`Song already requested by ${result.rows[0].requester}. State: ${result.rows[0].state}`] + + // Check whether the URL has already been requested + var existsResult = await checkRequestExists(url) + if (existsResult) { + console.log(existsResult); + return [200,`Song already requested by ${existsResult.rows[0].requester}. State: ${existsResult.rows[0].state}`] } + + // Check whether the user has hit their rate limit + var rateLimitQuery = Object.assign(queries.getRateLimitStatus, { values: [requester] }); + var rateLimitResult = (await db.query(rateLimitQuery)).rows[0]; + if (rateLimitResult.status) { + return [429,`You have reached your maximum of ${rateLimitResult.max} requests per day. + Please try again later.\n + Tip: Removing one of your requests from the past 24 hours by retracting your vote will allow you to replace it with another.`]; + } + + // Add the request var query = Object.assign(queries.addRequest, { values: [url,requester] }); return db.query(query) .then(async () => { From 8b49d51e08e947a99588608f7d27a34cea177af7 Mon Sep 17 00:00:00 2001 From: Dessa Simpson Date: Sun, 6 Jun 2021 23:20:47 -0700 Subject: [PATCH 3/6] Trim whitespace from URL on client side Fixes #32 --- public/main.js | 1 + 1 file changed, 1 insertion(+) diff --git a/public/main.js b/public/main.js index a5fc0dd..e9fee5d 100644 --- a/public/main.js +++ b/public/main.js @@ -75,6 +75,7 @@ function updateTable() { function applyUrlTransforms(url) { console.log("Begin applyUrlTransforms:" + url); + url = url.trim(); if (url.match(/^https?:\/\/(www\.)?youtu(\.be|be\.com)\//)) { // Youtube console.log("Youtube"); var videoid = ""; From 90f672b2888d8c72bf819f49ca861b817f4f50ee Mon Sep 17 00:00:00 2001 From: Dessa Simpson Date: Sun, 6 Jun 2021 23:32:53 -0700 Subject: [PATCH 4/6] Bump version to v0.8 --- db/00-version.sql | 2 +- db/upgrade/v0.7-v0.8.sql | 5 +++-- src/version.ts | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/db/00-version.sql b/db/00-version.sql index 23be6fa..6686897 100644 --- a/db/00-version.sql +++ b/db/00-version.sql @@ -8,4 +8,4 @@ CREATE OR REPLACE FUNCTION get_version() RETURNS VARCHAR AS $$SELECT major || '.' || minor FROM version $$ LANGUAGE SQL; -INSERT INTO version (major,minor) VALUES (0,7); +INSERT INTO version (major,minor) VALUES (0,8); diff --git a/db/upgrade/v0.7-v0.8.sql b/db/upgrade/v0.7-v0.8.sql index c5d5ab7..af83dae 100644 --- a/db/upgrade/v0.7-v0.8.sql +++ b/db/upgrade/v0.7-v0.8.sql @@ -1,5 +1,7 @@ BEGIN; +UPDATE version SET minor = 8; + ALTER TABLE config ALTER COLUMN normaluservotepoints SET DEFAULT 10, ALTER COLUMN followervotepoints SET DEFAULT 50, @@ -30,5 +32,4 @@ CREATE OR REPLACE VIEW ratelimit_vw AS )) AS ratelimit(reqcount); -ROLLBACK; ---COMMIT; +COMMIT; diff --git a/src/version.ts b/src/version.ts index 2751ddf..58b3b13 100644 --- a/src/version.ts +++ b/src/version.ts @@ -5,7 +5,7 @@ import pg from "pg"; import db from "./db"; var versionMajor = 0; -var versionMinor = 7; +var versionMinor = 8; var versionPatch = 0; export function getVersion() { From ac953b0bb5384ab1d852047d5699672681a78f34 Mon Sep 17 00:00:00 2001 From: Dessa Simpson Date: Sun, 27 Feb 2022 17:16:48 -0700 Subject: [PATCH 5/6] Uncomment debug lines --- src/cronjobs/processBans.ts | 8 ++++---- src/cronjobs/processSubscriptions.ts | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/cronjobs/processBans.ts b/src/cronjobs/processBans.ts index 2316612..a47a2fa 100644 --- a/src/cronjobs/processBans.ts +++ b/src/cronjobs/processBans.ts @@ -9,8 +9,8 @@ export async function processBans(streamer: twitch.StreamerUserIdTokenPair) { await dbconn.query("DELETE FROM bans"); var response = await twitch.streamerApiRequest(streamer, `/moderation/banned?broadcaster_id=${streamer.userid}&first=100`); - //log(LogLevel.DEBUG,"Ban API response:") - //log(LogLevel.DEBUG,JSON.stringify(response,null,2)); + log(LogLevel.DEBUG,"Ban API response:") + log(LogLevel.DEBUG,JSON.stringify(response,null,2)); while (true) { var insertBanQuery = "INSERT INTO bans (userid) VALUES "; var banRow = 0; // Used for $1, $2, etc. in parameterized query @@ -37,8 +37,8 @@ export async function processBans(streamer: twitch.StreamerUserIdTokenPair) { var oldFirstUserid = response.data[0].user_id; response = await twitch.streamerApiRequest(streamer, `/moderation/banned?broadcaster_id=${streamer.userid}&after=${response.pagination.cursor}&first=100`); - //log(LogLevel.DEBUG,"Ban API response:"); - //log(LogLevel.DEBUG,JSON.stringify(response,null,2)); + log(LogLevel.DEBUG,"Ban API response:"); + log(LogLevel.DEBUG,JSON.stringify(response,null,2)); // Work around broken api endpoint giving a cursor referring to the // current page, causing an infinite loop if (oldFirstUserid == response.data[0].user_id) break; diff --git a/src/cronjobs/processSubscriptions.ts b/src/cronjobs/processSubscriptions.ts index 3e250df..980e69b 100644 --- a/src/cronjobs/processSubscriptions.ts +++ b/src/cronjobs/processSubscriptions.ts @@ -9,8 +9,8 @@ export async function processSubscriptions(streamer: twitch.StreamerUserIdTokenP await dbconn.query("DELETE FROM subscriptions"); var response = await twitch.streamerApiRequest(streamer, `/subscriptions?broadcaster_id=${streamer.userid}&first=100`); - //log(LogLevel.DEBUG,"Subscriptions API response:") - //log(LogLevel.DEBUG,JSON.stringify(response,null,2)); + log(LogLevel.DEBUG,"Subscriptions API response:") + log(LogLevel.DEBUG,JSON.stringify(response,null,2)); while (true) { var insertSubscriptionQuery = "INSERT INTO subscriptions (userid) VALUES "; var subscriptionRow = 0; // Used for $1, $2, etc. in parameterized query @@ -36,8 +36,8 @@ export async function processSubscriptions(streamer: twitch.StreamerUserIdTokenP response = await twitch.streamerApiRequest(streamer, `/subscriptions?broadcaster_id=${streamer.userid}&after=${response.pagination.cursor}&first=100`); if (oldFirstUserid == response.data[0].user_id) break; - //log(LogLevel.DEBUG,"Subscription API response:"); - //log(LogLevel.DEBUG,JSON.stringify(response,null,2)); + log(LogLevel.DEBUG,"Subscription API response:"); + log(LogLevel.DEBUG,JSON.stringify(response,null,2)); } else { break; } From dd423bdb344bea065f421c97329323ca97cf3675 Mon Sep 17 00:00:00 2001 From: Dessa Simpson Date: Mon, 28 Feb 2022 09:34:18 -0700 Subject: [PATCH 6/6] Fix empty subscriptions page crash Fix issue where when Twitch returns an empty page of subscribers, the app crashes due to trying to get response.data[0].user_id where response.data[0] is undefined Also adds some debug logging --- src/cronjobs/processSubscriptions.ts | 3 ++- src/twitch.ts | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cronjobs/processSubscriptions.ts b/src/cronjobs/processSubscriptions.ts index 980e69b..5c53a10 100644 --- a/src/cronjobs/processSubscriptions.ts +++ b/src/cronjobs/processSubscriptions.ts @@ -35,9 +35,9 @@ export async function processSubscriptions(streamer: twitch.StreamerUserIdTokenP var oldFirstUserid = response.data[0].user_id; response = await twitch.streamerApiRequest(streamer, `/subscriptions?broadcaster_id=${streamer.userid}&after=${response.pagination.cursor}&first=100`); - if (oldFirstUserid == response.data[0].user_id) break; log(LogLevel.DEBUG,"Subscription API response:"); log(LogLevel.DEBUG,JSON.stringify(response,null,2)); + if (response.data.length === 0 || oldFirstUserid === response.data[0].user_id) break; } else { break; } @@ -45,6 +45,7 @@ export async function processSubscriptions(streamer: twitch.StreamerUserIdTokenP await dbconn.query("CALL update_scores()"); await dbconn.query('COMMIT'); } catch (e) { + log(LogLevel.ERROR,"cronjobs.processSubscriptions: Exception: " + e); log(LogLevel.ERROR,"cronjobs.processSubscriptions: Exception thrown; rolling back"); await dbconn.query('ROLLBACK'); throw(e); diff --git a/src/twitch.ts b/src/twitch.ts index 0569bee..494bc1a 100644 --- a/src/twitch.ts +++ b/src/twitch.ts @@ -53,6 +53,7 @@ export async function apiRequest(tokens: TokenPair, endpoint: string): Promise < return fetch("https://api.twitch.tv/helix" + endpoint, { headers: headers }) .then(async (res: FetchResponse) => { if (res.status == 200) { + log(LogLevel.DEBUG,"twitch.apiRequest: Request returned 200 for " + endpoint); return res.json(); } else { log(LogLevel.WARNING,"twitch.apiRequest: Failed API request (pre-refresh):");