Fix empty subscriptions page crash

Fix issue where when Twitch returns an empty page of subscribers, the
app crashes due to trying to get response.data[0].user_id where
response.data[0] is undefined

Also adds some debug logging

RELEASE.md

@@ -12,9 +12,19 @@ If the answer to any of the above is yes, then the release MUST be a major or mi
 - [ ] Add a commit which adds a database upgrade script at `db/upgrade/[oldversion]-[newversion].sql`
   - ALWAYS use a transaction for the entirety of this file
   - At minimum, DB version must be bumped
-- [ ] Add a commit which bumps the version in `app/version.ts` and `db/00-version.sql`
+  - Test the upgrade script as follows:
+    ```
+    git checkout [previous release tag]
+    docker-compose down
+    docker-compose up
+    psql -h 0 -U postgres < db/upgrade/v[previous]-v[current].sql
+    ```
+    Update the version in `src/version.ts` and verify the app works as expected.
+- [ ] Add a commit which bumps the version in `src/version.ts` and `db/00-version.sql`, entitled `Bump version to vMAJOR.MINOR`
 - [ ] Tag the latest commit with `vMAJOR.MINOR`
+- [ ] Write release notes
 
 ## Patch Releases
-- [ ] Add a commit which bumps the patch level in `app/version.ts`
+- [ ] Add a commit which bumps the patch level in `src/version.ts`, entitled `Bump version to vMAJOR.MINOR.PATCH`
 - [ ] Tag the latest commit with `vMAJOR.MINOR.PATCH`
+- [ ] Write release notes

db/00-version.sql

@@ -8,4 +8,4 @@ CREATE OR REPLACE FUNCTION get_version() RETURNS VARCHAR
 	AS $$SELECT major || '.' || minor FROM version $$
 	LANGUAGE SQL;
 
-INSERT INTO version (major,minor) VALUES (0,7);
+INSERT INTO version (major,minor) VALUES (0,8);

db/05-config.sql

@@ -1,12 +1,14 @@
 CREATE TABLE config (
 	rowlock bool DEFAULT TRUE UNIQUE NOT NULL CHECK (rowlock = TRUE),
-	normaluservotepoints int NOT NULL,
-	followervotepoints int NOT NULL,
-	subscribervotepoints int NOT NULL,
-	title varchar NOT NULL,
-	colors jsonb NOT NULL,
+	normaluservotepoints int NOT NULL DEFAULT 10,
+	followervotepoints int NOT NULL DEFAULT 50,
+	subscribervotepoints int NOT NULL DEFAULT 100,
+	normaluserratelimit int NOT NULL DEFAULT 1,
+	followerratelimit int NOT NULL DEFAULT 2,
+	subscriberratelimit int NOT NULL DEFAULT 3,
+	title varchar NOT NULL DEFAULT '{username}''s Learn Request Queue',
+	colors jsonb NOT NULL DEFAULT '{"bg": {"primary": "#444444","table": "#282828","navbar": "#666666","error": "#ff0000"},"fg": {"primary": "#dddddd","ahover": "#ffffff","title": "#eeeeee"}}',
 	PRIMARY KEY (rowLock)
 );
 
-INSERT INTO config (normalUserVotePoints,followerVotePoints,subscriberVotePoints,title,colors)
-	VALUES (10,50,100,'{username}''s Learn Request Queue','{"bg": {"primary": "#444444","table": "#282828","navbar": "#666666","error": "#ff0000"},"fg": {"primary": "#dddddd","ahover": "#ffffff","title": "#eeeeee"}}');
+INSERT INTO config (rowlock) VALUES (true);

db/90-views.sql

@@ -60,3 +60,22 @@ CREATE OR REPLACE VIEW streamer_user_vw AS
 	SELECT users.userid as userid, users.displayname as displayname, users.imageurl as imageurl FROM streamer
 		LEFT JOIN users
 			ON streamer.userid = users.userid;
+
+CREATE OR REPLACE VIEW ratelimit_vw AS
+	SELECT users.userid,COALESCE(count,0),ratelimit.reqcount AS max,COALESCE(count,0) >= ratelimit.reqcount AS status
+		FROM users
+		LEFT JOIN (SELECT requester,COUNT(url)
+			FROM requests
+			WHERE reqtimestamp > (now() - '24 hours'::interval)
+			GROUP BY requests.requester
+		) AS requests ON users.userid = requests.requester
+		LEFT JOIN follows ON requests.requester = follows.userid
+		LEFT JOIN subscriptions ON requests.requester = subscriptions.userid
+		CROSS JOIN config
+		CROSS JOIN LATERAL (VALUES (
+			CASE
+				WHEN follows.userid IS NULL AND subscriptions.userid IS NULL THEN config.normaluserratelimit
+				WHEN follows.userid IS NOT NULL AND subscriptions.userid IS NULL THEN config.followerratelimit
+				WHEN subscriptions.userid IS NOT NULL THEN config.subscriberratelimit
+			END
+		)) AS ratelimit(reqcount);

db/upgrade/v0.7-v0.8.sql

@@ -0,0 +1,35 @@
+BEGIN;
+
+UPDATE version SET minor = 8;
+
+ALTER TABLE config
+	ALTER COLUMN normaluservotepoints SET DEFAULT 10,
+	ALTER COLUMN followervotepoints SET DEFAULT 50,
+	ALTER COLUMN subscribervotepoints SET DEFAULT 100,
+	ALTER COLUMN title SET DEFAULT '{username}''s Learn Request Queue',
+	ALTER COLUMN colors SET DEFAULT '{"bg": {"primary": "#444444","table": "#282828","navbar": "#666666","error": "#ff0000"},"fg": {"primary": "#dddddd","ahover": "#ffffff","title": "#eeeeee"}}',
+	ADD COLUMN normaluserratelimit int NOT NULL DEFAULT 1,
+	ADD COLUMN followerratelimit int NOT NULL DEFAULT 2,
+	ADD COLUMN subscriberratelimit int NOT NULL DEFAULT 3;
+
+CREATE OR REPLACE VIEW ratelimit_vw AS
+	SELECT users.userid,COALESCE(count,0),ratelimit.reqcount AS max,COALESCE(count,0) >= ratelimit.reqcount AS status
+		FROM users
+		LEFT JOIN (SELECT requester,COUNT(url)
+			FROM requests
+			WHERE reqtimestamp > (now() - '24 hours'::interval)
+			GROUP BY requests.requester
+		) AS requests ON users.userid = requests.requester
+		LEFT JOIN follows ON requests.requester = follows.userid
+		LEFT JOIN subscriptions ON requests.requester = subscriptions.userid
+		CROSS JOIN config
+		CROSS JOIN LATERAL (VALUES (
+			CASE
+				WHEN follows.userid IS NULL AND subscriptions.userid IS NULL THEN config.normaluserratelimit
+				WHEN follows.userid IS NOT NULL AND subscriptions.userid IS NULL THEN config.followerratelimit
+				WHEN subscriptions.userid IS NOT NULL THEN config.subscriberratelimit
+			END
+		)) AS ratelimit(reqcount);
+
+
+COMMIT;

public/main.js

@@ -75,6 +75,7 @@ function updateTable() {
 
 function applyUrlTransforms(url) {
 	console.log("Begin applyUrlTransforms:" + url);
+	url = url.trim();
 	if (url.match(/^https?:\/\/(www\.)?youtu(\.be|be\.com)\//)) { // Youtube
 		console.log("Youtube");
 		var videoid = "";

src/cronjobs/processBans.ts

@@ -9,8 +9,8 @@ export async function processBans(streamer: twitch.StreamerUserIdTokenPair) {
 		await dbconn.query("DELETE FROM bans");
 		var response = await twitch.streamerApiRequest(streamer,
 			`/moderation/banned?broadcaster_id=${streamer.userid}&first=100`);
-		//log(LogLevel.DEBUG,"Ban API response:")
-		//log(LogLevel.DEBUG,JSON.stringify(response,null,2));
+		log(LogLevel.DEBUG,"Ban API response:")
+		log(LogLevel.DEBUG,JSON.stringify(response,null,2));
 		while (true) {
 			var insertBanQuery = "INSERT INTO bans (userid) VALUES ";
 			var banRow = 0; // Used for $1, $2, etc. in parameterized query
@@ -37,8 +37,8 @@ export async function processBans(streamer: twitch.StreamerUserIdTokenPair) {
 				var oldFirstUserid = response.data[0].user_id;
 				response = await twitch.streamerApiRequest(streamer,
 					`/moderation/banned?broadcaster_id=${streamer.userid}&after=${response.pagination.cursor}&first=100`);
-				//log(LogLevel.DEBUG,"Ban API response:");
-				//log(LogLevel.DEBUG,JSON.stringify(response,null,2));
+				log(LogLevel.DEBUG,"Ban API response:");
+				log(LogLevel.DEBUG,JSON.stringify(response,null,2));
 				// Work around broken api endpoint giving a cursor referring to the
 				// current page, causing an infinite loop
 				if (oldFirstUserid == response.data[0].user_id) break;

src/cronjobs/processSubscriptions.ts

@@ -9,8 +9,8 @@ export async function processSubscriptions(streamer: twitch.StreamerUserIdTokenP
 		await dbconn.query("DELETE FROM subscriptions");
 		var response = await twitch.streamerApiRequest(streamer,
 			`/subscriptions?broadcaster_id=${streamer.userid}&first=100`);
-		//log(LogLevel.DEBUG,"Subscriptions API response:")
-		//log(LogLevel.DEBUG,JSON.stringify(response,null,2));
+		log(LogLevel.DEBUG,"Subscriptions API response:")
+		log(LogLevel.DEBUG,JSON.stringify(response,null,2));
 		while (true) {
 			var insertSubscriptionQuery = "INSERT INTO subscriptions (userid) VALUES ";
 			var subscriptionRow = 0; // Used for $1, $2, etc. in parameterized query
@@ -35,9 +35,9 @@ export async function processSubscriptions(streamer: twitch.StreamerUserIdTokenP
 				var oldFirstUserid = response.data[0].user_id;
 				response = await twitch.streamerApiRequest(streamer,
 					`/subscriptions?broadcaster_id=${streamer.userid}&after=${response.pagination.cursor}&first=100`);
-				if (oldFirstUserid == response.data[0].user_id) break;
-				//log(LogLevel.DEBUG,"Subscription API response:");
-				//log(LogLevel.DEBUG,JSON.stringify(response,null,2));
+				log(LogLevel.DEBUG,"Subscription API response:");
+				log(LogLevel.DEBUG,JSON.stringify(response,null,2));
+				if (response.data.length === 0 || oldFirstUserid === response.data[0].user_id) break;
 			} else {
 				break;
 			}
@@ -45,6 +45,7 @@ export async function processSubscriptions(streamer: twitch.StreamerUserIdTokenP
 		await dbconn.query("CALL update_scores()");
 		await dbconn.query('COMMIT');
 	} catch (e) {
+		log(LogLevel.ERROR,"cronjobs.processSubscriptions: Exception: " + e);
 		log(LogLevel.ERROR,"cronjobs.processSubscriptions: Exception thrown; rolling back");
 		await dbconn.query('ROLLBACK');
 		throw(e);

src/queries.ts

@@ -130,3 +130,8 @@ export const updateCronJobLastSuccess = {
 	name: "updateCronJobLastSuccess",
 	text: "UPDATE cron SET lastSuccess = now() WHERE jobName = $1"
 }
+
+export const getRateLimitStatus = {
+	name: "getRateLimitStatus",
+	text: "SELECT * FROM ratelimit_vw WHERE userid = $1"
+}

src/requests.ts

@@ -87,6 +87,7 @@ async function retrieveYoutubeMetadata(url: string) {
 }
 
 export async function addRequest(url: string, requester: string): Promise<[number,string]> {
+	// Check that URL is of an accepted format
 	var validUrl = false;
 	for (var regex of validUrlRegexes) {
 		if (regex.test(url)) {
@@ -95,11 +96,24 @@ export async function addRequest(url: string, requester: string): Promise<[numbe
 		}
 	}
 	if (!validUrl) return [400, "Invalid song URL."];
-	var result = await checkRequestExists(url)
-	if (result) {
-		console.log(result);
-		return [200,`Song already requested by ${result.rows[0].requester}. State: ${result.rows[0].state}`]
+
+	// Check whether the URL has already been requested
+	var existsResult = await checkRequestExists(url)
+	if (existsResult) {
+		console.log(existsResult);
+		return [200,`Song already requested by ${existsResult.rows[0].requester}. State: ${existsResult.rows[0].state}`]
 	}
+
+	// Check whether the user has hit their rate limit
+	var rateLimitQuery = Object.assign(queries.getRateLimitStatus, { values: [requester] });
+	var rateLimitResult = (await db.query(rateLimitQuery)).rows[0];
+	if (rateLimitResult.status) {
+		return [429,`You have reached your maximum of ${rateLimitResult.max} requests per day.
+			Please try again later.\n
+			Tip: Removing one of your requests from the past 24 hours by retracting your vote will allow you to replace it with another.`];
+	}
+
+	// Add the request
 	var query = Object.assign(queries.addRequest, { values: [url,requester] });
 	return db.query(query)
 		.then(async () => {

src/twitch.ts

@@ -53,6 +53,7 @@ export async function apiRequest(tokens: TokenPair, endpoint: string): Promise <
 	return fetch("https://api.twitch.tv/helix" + endpoint, { headers: headers })
 		.then(async (res: FetchResponse) => {
 			if (res.status == 200) {
+				log(LogLevel.DEBUG,"twitch.apiRequest: Request returned 200 for " + endpoint);
 				return res.json();
 			} else {
 				log(LogLevel.WARNING,"twitch.apiRequest: Failed API request (pre-refresh):");

src/version.ts

@@ -5,7 +5,7 @@ import pg from "pg";
 import db from "./db";
 
 var versionMajor = 0;
-var versionMinor = 7;
+var versionMinor = 8;
 var versionPatch = 0;
 
 export function getVersion() {