Fix empty subscriptions page crash

Fix issue where when Twitch returns an empty page of subscribers, the
app crashes due to trying to get response.data[0].user_id where
response.data[0] is undefined

Also adds some debug logging

RELEASE.md

@@ -12,9 +12,19 @@ If the answer to any of the above is yes, then the release MUST be a major or mi
 - [ ] Add a commit which adds a database upgrade script at `db/upgrade/[oldversion]-[newversion].sql`
   - ALWAYS use a transaction for the entirety of this file
   - At minimum, DB version must be bumped
-- [ ] Add a commit which bumps the version in `app/version.ts` and `db/00-version.sql`
+  - Test the upgrade script as follows:
+    ```
+    git checkout [previous release tag]
+    docker-compose down
+    docker-compose up
+    psql -h 0 -U postgres < db/upgrade/v[previous]-v[current].sql
+    ```
+    Update the version in `src/version.ts` and verify the app works as expected.
+- [ ] Add a commit which bumps the version in `src/version.ts` and `db/00-version.sql`, entitled `Bump version to vMAJOR.MINOR`
 - [ ] Tag the latest commit with `vMAJOR.MINOR`
+- [ ] Write release notes
 
 ## Patch Releases
-- [ ] Add a commit which bumps the patch level in `app/version.ts`
+- [ ] Add a commit which bumps the patch level in `src/version.ts`, entitled `Bump version to vMAJOR.MINOR.PATCH`
 - [ ] Tag the latest commit with `vMAJOR.MINOR.PATCH`
+- [ ] Write release notes

db/00-version.sql

@@ -8,4 +8,4 @@ CREATE OR REPLACE FUNCTION get_version() RETURNS VARCHAR
 	AS $$SELECT major || '.' || minor FROM version $$
 	LANGUAGE SQL;
 
-INSERT INTO version (major,minor) VALUES (0,4);
+INSERT INTO version (major,minor) VALUES (0,8);

db/05-config.sql

@@ -1,12 +1,14 @@
 CREATE TABLE config (
 	rowlock bool DEFAULT TRUE UNIQUE NOT NULL CHECK (rowlock = TRUE),
-	normaluservotepoints int NOT NULL,
-	followervotepoints int NOT NULL,
-	subscribervotepoints int NOT NULL,
-	title varchar NOT NULL,
-	colors jsonb NOT NULL,
+	normaluservotepoints int NOT NULL DEFAULT 10,
+	followervotepoints int NOT NULL DEFAULT 50,
+	subscribervotepoints int NOT NULL DEFAULT 100,
+	normaluserratelimit int NOT NULL DEFAULT 1,
+	followerratelimit int NOT NULL DEFAULT 2,
+	subscriberratelimit int NOT NULL DEFAULT 3,
+	title varchar NOT NULL DEFAULT '{username}''s Learn Request Queue',
+	colors jsonb NOT NULL DEFAULT '{"bg": {"primary": "#444444","table": "#282828","navbar": "#666666","error": "#ff0000"},"fg": {"primary": "#dddddd","ahover": "#ffffff","title": "#eeeeee"}}',
 	PRIMARY KEY (rowLock)
 );
 
-INSERT INTO config (normalUserVotePoints,followerVotePoints,subscriberVotePoints,title,colors)
-	VALUES (10,50,100,'{username}''s Learn Request Queue','{"bg": {"primary": "#444444","table": "#282828","navbar": "#666666","error": "#ff0000"},"fg": {"primary": "#dddddd","ahover": "#ffffff","title": "#eeeeee"}}');
+INSERT INTO config (rowlock) VALUES (true);

db/10-cron.sql

@@ -8,4 +8,6 @@ CREATE TABLE cron (
 
 INSERT INTO cron (jobName,runInterval) VALUES
 ('processBans','30 minutes'),
+('processFollows','30 minutes'),
+('processSubscriptions','30 minutes'),
 ('processEmptyMetadata','1 day');

db/10-users.sql

@@ -2,7 +2,5 @@ CREATE TABLE users (
 	userId int NOT NULL,
 	displayName varchar NOT NULL,
 	imageUrl varchar,
-	isFollower boolean NOT NULL DEFAULT FALSE,
-	isSubscriber boolean NOT NULL DEFAULT FALSE,
 	PRIMARY KEY (userId)
 );

db/15-streamer.sql

@@ -3,4 +3,4 @@ CREATE TABLE streamer (
 	tokenPair json,
 	PRIMARY KEY (userid),
 	FOREIGN KEY (userid) REFERENCES users(userid)
-)
+);

db/20-requests.sql

@@ -1,5 +1,5 @@
 CREATE TABLE requests (
-	url varchar NOT NULL UNIQUE,
+	url varchar NOT NULL,
 	requester int NOT NULL,
 	state varchar NOT NULL DEFAULT 'Requested',
 	reqTimestamp timestamptz NOT NULL DEFAULT NOW(),

db/30-votes.sql

@@ -1,6 +1,6 @@
 CREATE TABLE votes (
-	requestUrl varchar,
-	userId int,
+	requestUrl varchar NOT NULL,
+	userId int NOT NULL,
 	PRIMARY KEY (requestUrl,userId),
 	FOREIGN KEY (requestUrl) REFERENCES requests(url) ON DELETE CASCADE,
 	FOREIGN KEY (userId) REFERENCES users(userId) ON DELETE CASCADE

db/50-bans.sql

@@ -1,4 +1,4 @@
 CREATE TABLE bans (
 	userid integer,
 	PRIMARY KEY (userid)
-)
+);

db/50-follows.sql

@@ -0,0 +1,4 @@
+CREATE TABLE follows (
+	userid integer,
+	PRIMARY KEY (userid)
+);

db/50-subscriptions.sql

@@ -0,0 +1,4 @@
+CREATE TABLE subscriptions (
+	userid integer,
+	PRIMARY KEY (userid)
+);

db/90-views.sql

@@ -2,8 +2,7 @@ CREATE OR REPLACE VIEW requests_vw AS
 	SELECT url,COALESCE(videoTitle,url) AS title,displayName AS requester,imageUrl,state,score,reqTimestamp FROM requests
 		JOIN requestMetadata USING (url)
 		JOIN scores USING (url)
-		JOIN users ON requests.requester = users.userid
-		ORDER BY score DESC, reqTimestamp ASC;
+		JOIN users ON requests.requester = users.userid;
 
 CREATE OR REPLACE FUNCTION get_requests_voted(votinguserid int)
 	RETURNS TABLE (
@@ -21,8 +20,7 @@ CREATE OR REPLACE FUNCTION get_requests_voted(votinguserid int)
 		SELECT url,title,requester,imageUrl,state,score,reqTimestamp,
 				(CASE WHEN votes.userid IS NULL THEN FALSE ELSE TRUE END) AS voted
 			FROM requests_vw
-			LEFT JOIN votes ON (requests_vw.url = votes.requesturl AND votes.userid = votinguserid)
-			ORDER BY score DESC, reqTimestamp ASC;
+			LEFT JOIN votes ON (requests_vw.url = votes.requesturl AND votes.userid = votinguserid);
 	$$;
 
 /*
@@ -40,18 +38,20 @@ CREATE OR REPLACE VIEW vote_score_vw AS
 		COUNT(votes.requesturl) AS count,
 		COALESCE(
 			SUM(CASE
-				WHEN users.isfollower = FALSE AND users.issubscriber = FALSE
+				WHEN follows.userid IS NULL AND subscriptions.userid IS NULL
+					AND votes.userid IS NOT NULL
 				THEN votepoints.normaluservotepoints
-				WHEN users.isfollower = TRUE AND users.issubscriber = FALSE
+				WHEN follows.userid IS NOT NULL AND subscriptions.userid IS NULL
 				THEN votepoints.followervotepoints
-				WHEN users.issubscriber = TRUE
+				WHEN subscriptions.userid IS NOT NULL
 				THEN votepoints.subscribervotepoints
 			END), 0
 		) AS votescore
 		FROM requests
 		LEFT JOIN votes ON votes.requesturl = requests.url
-		LEFT JOIN users on votes.userid = users.userid
-		LEFT JOIN bans ON users.userid = bans.userid
+		LEFT JOIN bans ON votes.userid = bans.userid
+		LEFT JOIN follows ON votes.userid = follows.userid
+		LEFT JOIN subscriptions ON votes.userid = subscriptions.userid
 		CROSS JOIN votepoints
 		WHERE bans.userid IS NULL
 		GROUP BY url;
@@ -60,3 +60,22 @@ CREATE OR REPLACE VIEW streamer_user_vw AS
 	SELECT users.userid as userid, users.displayname as displayname, users.imageurl as imageurl FROM streamer
 		LEFT JOIN users
 			ON streamer.userid = users.userid;
+
+CREATE OR REPLACE VIEW ratelimit_vw AS
+	SELECT users.userid,COALESCE(count,0),ratelimit.reqcount AS max,COALESCE(count,0) >= ratelimit.reqcount AS status
+		FROM users
+		LEFT JOIN (SELECT requester,COUNT(url)
+			FROM requests
+			WHERE reqtimestamp > (now() - '24 hours'::interval)
+			GROUP BY requests.requester
+		) AS requests ON users.userid = requests.requester
+		LEFT JOIN follows ON requests.requester = follows.userid
+		LEFT JOIN subscriptions ON requests.requester = subscriptions.userid
+		CROSS JOIN config
+		CROSS JOIN LATERAL (VALUES (
+			CASE
+				WHEN follows.userid IS NULL AND subscriptions.userid IS NULL THEN config.normaluserratelimit
+				WHEN follows.userid IS NOT NULL AND subscriptions.userid IS NULL THEN config.followerratelimit
+				WHEN subscriptions.userid IS NOT NULL THEN config.subscriberratelimit
+			END
+		)) AS ratelimit(reqcount);

db/95-procedures.sql

@@ -16,6 +16,13 @@ CREATE OR REPLACE PROCEDURE add_request(url varchar,requester int)
 		CALL update_scores();
 	$$;
 
+CREATE OR REPLACE PROCEDURE clear_zero_votes()
+	LANGUAGE SQL
+	AS $$
+		DELETE FROM requests WHERE NOT EXISTS
+			(SELECT FROM votes WHERE requests.url = votes.requesturl);
+	$$;
+
 CREATE OR REPLACE PROCEDURE add_vote(url varchar,voteuser int)
 	LANGUAGE SQL
 	AS $$
@@ -28,6 +35,7 @@ CREATE OR REPLACE PROCEDURE delete_vote(url varchar,voteuser int)
 	AS $$
 		DELETE FROM votes WHERE requesturl = url AND userid = voteuser;
 		CALL update_scores();
+		CALL clear_zero_votes();
 	$$;
 
 CREATE OR REPLACE PROCEDURE update_request_score_modifier(updateurl varchar, scoreDiff int)
@@ -36,3 +44,11 @@ CREATE OR REPLACE PROCEDURE update_request_score_modifier(updateurl varchar, sco
 		UPDATE scores SET scoreModifier = scoreModifier + scoreDiff WHERE url = updateurl;
 		CALL update_scores();
 	$$;
+
+CREATE OR REPLACE PROCEDURE update_vote_points(normaluser int, follower int, subscriber int)
+	LANGUAGE SQL
+	AS $$
+		UPDATE config SET normaluservotepoints = normaluser,
+			followervotepoints = follower, subscribervotepoints = subscriber;
+		CALL update_scores();
+	$$;

db/testdata.sql

@@ -1,8 +1,16 @@
-INSERT INTO users (userid,displayName,isFollower,isSubscriber) VALUES
-	(001,'TestUser',false,false),
-	(002,'TestFollower',true,false),
-	(003,'TestSubscriber',true,true),
-	(004,'TestSubNonFollower',false,true);
+INSERT INTO users (userid,displayName) VALUES
+	(001,'TestUser'),
+	(002,'TestFollower'),
+	(003,'TestSubscriber'),
+	(004,'TestSubNonFollower');
+
+INSERT INTO follows (userid) VALUES
+	(002),
+	(003);
+
+INSERT INTO subscriptions (userid) VALUES
+	(003),
+	(004);
 
 CALL add_request('https://www.youtube.com/watch?v=dQw4w9WgXcQ',001);
 CALL add_request('https://www.youtube.com/watch?v=C5oeWHngDS4',002);

db/upgrade/v0.4-0.5.sql

@@ -0,0 +1,53 @@
+BEGIN;
+
+UPDATE version SET minor = 5;
+
+CREATE OR REPLACE PROCEDURE update_vote_points(normaluser int, follower int, subscriber int)
+	LANGUAGE SQL
+	AS $$
+		UPDATE config SET normaluservotepoints = normaluser,
+			followervotepoints = follower, subscribervotepoints = subscriber;
+		CALL update_scores();
+	$$;
+
+CREATE TABLE follows (
+	userid integer,
+	PRIMARY KEY (userid)
+);
+
+CREATE TABLE subscriptions (
+	userid integer,
+	PRIMARY KEY (userid)
+);
+
+CREATE OR REPLACE VIEW vote_score_vw AS
+	WITH votepoints AS (SELECT normaluservotepoints, followervotepoints, subscribervotepoints FROM config)
+	SELECT requests.url AS url,
+		COUNT(votes.requesturl) AS count,
+		COALESCE(
+			SUM(CASE
+				WHEN follows.userid IS NULL AND subscriptions.userid IS NULL
+					AND votes.userid IS NOT NULL
+				THEN votepoints.normaluservotepoints
+				WHEN follows.userid IS NOT NULL AND subscriptions.userid IS NULL
+				THEN votepoints.followervotepoints
+				WHEN subscriptions.userid IS NOT NULL
+				THEN votepoints.subscribervotepoints
+			END), 0
+		) AS votescore
+		FROM requests
+		LEFT JOIN votes ON votes.requesturl = requests.url
+		LEFT JOIN bans ON votes.userid = bans.userid
+		LEFT JOIN follows ON votes.userid = follows.userid
+		LEFT JOIN subscriptions ON votes.userid = subscriptions.userid
+		CROSS JOIN votepoints
+		WHERE bans.userid IS NULL
+		GROUP BY url;
+
+ALTER TABLE users DROP COLUMN isfollower, DROP COLUMN issubscriber;
+
+INSERT INTO cron (jobName,runInterval) VALUES
+('processFollows','30 minutes'),
+('processSubscriptions','30 minutes');
+
+COMMIT;

db/upgrade/v0.5-v0.6.sql

@@ -0,0 +1,5 @@
+BEGIN;
+
+UPDATE version SET minor = 6;
+
+COMMIT;

db/upgrade/v0.6-v0.7.sql

@@ -0,0 +1,24 @@
+BEGIN;
+
+UPDATE version SET minor = 7;
+
+ALTER TABLE votes
+	ALTER COLUMN requestUrl SET NOT NULL,
+	ALTER COLUMN userId SET NOT NULL;
+
+CREATE OR REPLACE PROCEDURE clear_zero_votes()
+	LANGUAGE SQL
+	AS $$
+		DELETE FROM requests WHERE NOT EXISTS
+			(SELECT FROM votes WHERE requests.url = votes.requesturl);
+	$$;
+
+CREATE OR REPLACE PROCEDURE delete_vote(url varchar,voteuser int)
+	LANGUAGE SQL
+	AS $$
+		DELETE FROM votes WHERE requesturl = url AND userid = voteuser;
+		CALL update_scores();
+		CALL clear_zero_votes();
+	$$;
+
+COMMIT;

db/upgrade/v0.7-v0.8.sql

@@ -0,0 +1,35 @@
+BEGIN;
+
+UPDATE version SET minor = 8;
+
+ALTER TABLE config
+	ALTER COLUMN normaluservotepoints SET DEFAULT 10,
+	ALTER COLUMN followervotepoints SET DEFAULT 50,
+	ALTER COLUMN subscribervotepoints SET DEFAULT 100,
+	ALTER COLUMN title SET DEFAULT '{username}''s Learn Request Queue',
+	ALTER COLUMN colors SET DEFAULT '{"bg": {"primary": "#444444","table": "#282828","navbar": "#666666","error": "#ff0000"},"fg": {"primary": "#dddddd","ahover": "#ffffff","title": "#eeeeee"}}',
+	ADD COLUMN normaluserratelimit int NOT NULL DEFAULT 1,
+	ADD COLUMN followerratelimit int NOT NULL DEFAULT 2,
+	ADD COLUMN subscriberratelimit int NOT NULL DEFAULT 3;
+
+CREATE OR REPLACE VIEW ratelimit_vw AS
+	SELECT users.userid,COALESCE(count,0),ratelimit.reqcount AS max,COALESCE(count,0) >= ratelimit.reqcount AS status
+		FROM users
+		LEFT JOIN (SELECT requester,COUNT(url)
+			FROM requests
+			WHERE reqtimestamp > (now() - '24 hours'::interval)
+			GROUP BY requests.requester
+		) AS requests ON users.userid = requests.requester
+		LEFT JOIN follows ON requests.requester = follows.userid
+		LEFT JOIN subscriptions ON requests.requester = subscriptions.userid
+		CROSS JOIN config
+		CROSS JOIN LATERAL (VALUES (
+			CASE
+				WHEN follows.userid IS NULL AND subscriptions.userid IS NULL THEN config.normaluserratelimit
+				WHEN follows.userid IS NOT NULL AND subscriptions.userid IS NULL THEN config.followerratelimit
+				WHEN subscriptions.userid IS NOT NULL THEN config.subscriberratelimit
+			END
+		)) AS ratelimit(reqcount);
+
+
+COMMIT;

public/main.js

@@ -1,29 +1,52 @@
 var requestsDiv = document.getElementById("requests");
-var cronJobs = ['processBans']
+var cronJobs = ['processBans'];
+var currentPage = 1;
+var totalPages = 1;
+var count = document.getElementById("count").value;
+var sortBy = document.getElementById("sortBy").value;
+var sortDir = "desc";
 
-function getRequests(count,allRequests) {
-	var reqUrl;
-	if (allRequests) {
-		reqUrl = "/api/getAllRequests";
-	} else {
-		reqUrl = "/api/getRequests";
-	}
-	reqUrl += `?count=${count}`;
+function getRequests(offset,allRequests) {
+	if (allRequests) var reqUrl = "/api/getAllRequests";
+	else var reqUrl = "/api/getRequests";
+	reqUrl += `?count=${count}&offset=${offset}&sort=${sortBy}&sortDirection=${sortDir}`;
 	fetch(reqUrl)
 		.then(response => response.json())
 		.then(requests => {
-			window.requests = requests;
-			buildTable();
+			buildTable(requests);
 		});
 }
 
-function buildTable() {
-	var requestsDivHTML = '<table><tr><th class="request-link">Song</th><th class="request-requester">Requester</th><th class="request-score">Score</th>';
+function buildTable(requests) {
+	totalPages = Math.ceil(requests.total/document.getElementById("count").value);
+	document.getElementById("totalPages").innerText = totalPages;
+	if (currentPage <= 1) {
+		currentPage = 1;
+		document.getElementById("pageBtnFirst").disabled = true;
+		document.getElementById("pageBtnPrev").disabled = true;
+	} else {
+		document.getElementById("pageBtnFirst").disabled = false;
+		document.getElementById("pageBtnPrev").disabled = false;
+	}
+	if (currentPage >= totalPages) {
+		currentPage = totalPages;
+		document.getElementById("pageBtnLast").disabled = true;
+		document.getElementById("pageBtnNext").disabled = true;
+	} else {
+		document.getElementById("pageBtnLast").disabled = false;
+		document.getElementById("pageBtnNext").disabled = false;
+	}
+	document.getElementById("page").innerHTML = "";
+	for (i = 1; i <= totalPages; i++) {
+		document.getElementById("page").innerHTML += `<option value=${i}>${i}</option>`;
+	}
+	document.getElementById("page").value = currentPage;
+	var requestsDivHTML = '<table><tr><th class="request-link">Song Title</th><th class="request-requester">Requester</th><th class="request-score">Score</th>';
 	requestsDivHTML += '<th class="request-state">State</td>';
 	if (window.loggedIn) requestsDivHTML += '<th class="request-vote">Vote</td>';
 	if (window.isStreamer) requestsDivHTML += '<th class="request-update">Update</th>'
 	requestsDivHTML += "</tr>";
-	for (request of requests) {
+	for (request of requests.requests) {
 		requestsDivHTML += `<tr><td class="request-link"><a href="${request.url}" target="_blank">${request.title}</a></td>\
 			<td class="request-requester">${request.imageurl ? `<img src="${request.imageurl}" class="table-userpic"/>` : ''}${request.requester}</td>\
 			<td class="request-score">${request.score}</td>`;
@@ -45,12 +68,14 @@ function buildTable() {
 }
 
 function updateTable() {
-	allRequests = document.getElementById("allRequests").checked;
-	getRequests(document.getElementById("count").value,allRequests);
+	var offset = (currentPage - 1) * count;
+	var allRequests = document.getElementById("allRequests").checked;
+	getRequests(offset,allRequests);
 }
 
 function applyUrlTransforms(url) {
 	console.log("Begin applyUrlTransforms:" + url);
+	url = url.trim();
 	if (url.match(/^https?:\/\/(www\.)?youtu(\.be|be\.com)\//)) { // Youtube
 		console.log("Youtube");
 		var videoid = "";
@@ -66,6 +91,11 @@ function applyUrlTransforms(url) {
 	}
 }
 
+function goToPage(page) {
+	currentPage = parseInt(page,10);
+	updateTable();
+}
+
 function getColorObject() {
 	return {
 		bg: {
@@ -137,6 +167,7 @@ function showMessage(msg) {
 function openAddRequestModal() {
 	hideModals();
 	document.getElementById("modalBackground").style.display = "flex";
+	document.getElementById("addRequestModal").style.display = "block";
 }
 
 function openUpdateRequestModal(tr) {
@@ -205,7 +236,7 @@ function validateAndSubmitRequest() {
 			updateTable();
 			document.getElementById("addRequestUrl").value = "";
 			response.text().then((message) => {
-				closeAddRequestModal();
+				closeAllModals();
 				showMessage(message);
 			});
 		});
@@ -317,12 +348,27 @@ function updateColors(colors) {
 		});
 }
 
+function updateVotePoints(user,follower,subscriber) {
+	streamerSettingsErrReset();
+	fetch("/api/updateVotePoints", { method: 'POST', body: new URLSearchParams({
+		user: user,
+		follower: follower,
+		subscriber: subscriber
+	})})
+		.then(response => {
+			if (response.ok) {
+				location.reload();
+			} else {
+				response.text().then(streamerSettingsErr);
+			}
+		});
+}
+
 updateTable();
 
 document.addEventListener("keydown", function onEvent(event) {
     if (event.key === "Escape") {
-        closeMessageModal();
-        closeAddRequestModal();
+        closeAllModals();
     }
 });
 document.getElementById("modalBackground").addEventListener("click", (e) => { if (e.target === e.currentTarget) closeAllModals();});
@@ -332,5 +378,16 @@ for(state of validStates) {
 	var opt = document.createElement("option");
 	opt.text = state;
 	opt.value = state;
-	updateRequestStateSelect.add(opt)
+	updateRequestStateSelect.add(opt);
+}
+
+function toggleSortDir() {
+	if (window.sortDir == "desc") {
+		document.getElementById("sortDir").innerText = "↑";
+		window.sortDir = "asc";
+	} else {
+		document.getElementById("sortDir").innerText = "↓";
+		window.sortDir = "desc";
+	}
+	updateTable();
 }

public/style.css

@@ -111,6 +111,20 @@ div#nav-userpic {
 	text-align: right;
 }
 
+.tableSettings {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+}
+
+.tableSettings > span {
+  margin: auto;
+}
+
+#tableSettingsTop {
+  margin-bottom: 10px;
+}
+
 #modalBackground {
 	display: none;
 	position: fixed;
@@ -123,6 +137,7 @@ div#nav-userpic {
 	height: 100%;
 	background-color: #444;
 	background-color: #444a;
+  margin: 5px;
 }
 
 .modal {
@@ -185,3 +200,18 @@ div#nav-userpic {
 #deleteRequestLink {
   color: #f00;
 }
+
+#streamerSettingsMain {
+  max-height: 65vh;
+  overflow-y: scroll;
+}
+
+#votepoints {
+  padding: 0 1.5em;
+  display: flex;
+  justify-content: space-between;
+}
+
+#sortDir {
+  padding: 0;
+}

src/app.ts

@@ -42,31 +42,85 @@ app.use(session({
 
 // API
 app.get("/api/getRequests", async (request, response) => {
-	if (!request.session) {
-		throw new Error ("Missing request.session")
-	}
-	var requestCount = ( request.query.count ? parseInt(request.query.count as string, 10) : 5 );
+	if (!request.session) throw new Error ("Missing request.session");
 	await validateApiToken(request.session);
+	var requestCount = ( request.query.count ? parseInt(request.query.count as string, 10) : 5 );
+	var requestOffset = ( request.query.offset ? parseInt(request.query.offset as string, 10) : 0 );
+	var sortDirection = ( request.query.sortDirection == "asc" ? "ASC" : "DESC" );
+	var inverseSortDirection = ( sortDirection == "ASC" ? "DESC" : "ASC" );
+	switch (request.query.sort) {
+		case undefined: // Default sort by newest
+		case "timestamp":
+			var requestSort = `reqTimestamp ${sortDirection}`;
+			break;
+		case "score":
+			var requestSort = `score ${sortDirection}, reqTimestamp ${inverseSortDirection}`;
+			break;
+		case "title":
+			var requestSort = `title ${sortDirection}`
+			break;
+		case "requester":
+			var requestSort = `requester ${sortDirection}, title ${sortDirection}`
+			break;
+		default:
+			response.status(400);
+			response.send("Invalid sort");
+			return;
+	};
+	var requestsTotal = await requests.getRequestsTotal();
 	if (request.session.user) {
-		requests.getRequestsVoted(requestCount,request.session.user.id).then((val: Array<any>) => response.send(val))
+		requests.getRequestsVoted(requestCount,requestOffset,requestSort,request.session.user.id)
+			.then((val: Array<any>) => response.send({
+					total: requestsTotal,
+					requests: val
+				}))
 			.catch((e: any) => errorHandler(request,response,e));
 	} else {
-		requests.getRequests(requestCount).then((val: Array<any>) => response.send(val))
+		requests.getRequests(requestCount,requestOffset,requestSort)
+			.then((val: Array<any>) => response.send({
+					total: requestsTotal,
+					requests: val
+				}))
 			.catch((e: any) => errorHandler(request,response,e));
 	}
 });
 
 app.get("/api/getAllRequests", async (request, response) => {
-	if (!request.session) {
-		throw new Error ("Missing request.session")
-	}
-	var requestCount = ( request.query.count ? parseInt(request.query.count as string, 10) : 5 );
+	if (!request.session) throw new Error ("Missing request.session");
 	await validateApiToken(request.session);
+	var requestCount = ( request.query.count ? parseInt(request.query.count as string, 10) : 5 );
+	var requestOffset = ( request.query.offset ? parseInt(request.query.offset as string, 10) : 0 );
+	var sortDirection = ( request.query.sortDirection == "asc" ? "ASC" : "DESC" );
+	switch (request.query.sort) {
+		case undefined:
+		case "score":
+			var requestSort = `score ${sortDirection}, reqTimestamp ASC`;
+			break;
+		case "timestamp":
+			var requestSort = `reqTimestamp ${sortDirection}`;
+			break;
+		case "alpha":
+			var requestSort = `title ${sortDirection}`
+			break;
+		default:
+			response.status(400);
+			response.send("Invalid sort");
+			return;
+	}
+	var requestsTotal = await requests.getAllRequestsTotal();
 	if (request.session.user) {
-		requests.getAllRequestsVoted(requestCount,request.session.user.id).then((val: Array<any>) => response.send(val))
+		requests.getAllRequestsVoted(requestCount,requestOffset,requestSort,request.session.user.id)
+			.then((val: Array<any>) => response.send({
+					total: requestsTotal,
+					requests: val
+				}))
 			.catch((e: any) => errorHandler(request,response,e));
 	} else {
-		requests.getAllRequests(requestCount).then((val: Array<any>) => response.send(val))
+		requests.getAllRequests(requestCount,requestOffset,requestSort)
+			.then((val: Array<any>) => response.send({
+					total: requestsTotal,
+					requests: val
+				}))
 			.catch((e: any) => errorHandler(request,response,e));
 	}
 });
@@ -79,7 +133,8 @@ app.post("/api/addRequest", async (request, response) => {
 		response.send("Session expired; please log in again");
 		return;
 	}
-	var banned = await db.query(Object.assign(queries.checkBan, { values: [request.session.user.id] })).then((result: pg.QueryResult) => result.rowCount > 0);
+	var banned = await db.query(Object.assign(queries.checkBan, { values: [request.session.user.id] }))
+		.then((result: pg.QueryResult) => result.rowCount > 0);
 	if (banned) {
 		response.status(401);
 		response.send("You are banned; you may not add new requests.");
@@ -285,6 +340,44 @@ app.post("/api/updateColors", async (request, response) => {
 	response.send('Successfully updated colors');
 });
 
+app.post("/api/updateVotePoints", async (request, response) => {
+	if (request.session) await validateApiToken(request.session);
+	if (!request.session || !request.session.user) {
+		response.status(401);
+		response.send("Session expired; please log in again");
+		return;
+	}
+	var streamerid = await db.query(queries.getStreamerId).then((result: pg.QueryResult) => result.rows[0]['userid']);
+	if (request.session.user.id != streamerid) {
+		response.status(401);
+		response.send("You are not the streamer");
+		return;
+	}
+	if (!request.body.user) {
+		response.status(400);
+		response.send("Missing user");
+		return;
+	}
+	if (!request.body.follower) {
+		response.status(400);
+		response.send("Missing follower");
+		return;
+	}
+	if (!request.body.subscriber) {
+		response.status(400);
+		response.send("Missing subscriber");
+		return;
+	}
+	var user = request.body.user as number;
+	var follower = request.body.follower as number;
+	var subscriber = request.body.subscriber as number;
+	response.type('text/plain');
+	await db.query(Object.assign(queries.updateVotePoints,{ values: [user,follower,subscriber] }))
+	.catch((e: any) => errorHandler(request,response,e));
+	response.status(200);
+	response.send('Successfully updated page title');
+});
+
 app.post("/api/deleteRequest", async (request, response) => {
 	if (request.session) await validateApiToken(request.session);
 	if (!request.session || !request.session.user) {
@@ -439,10 +532,9 @@ app.get("/", async (request, response) => {
 			loggedIn: false,
 			clientId: config.twitchClientId,
 			urlPrefix: config.urlPrefix,
-			pageTitle: streamerConfig.title,
 			streamerName: streamerInfo['displayname'],
 			streamerProfilePicture: streamerInfo['imageurl'],
-			colors: streamerConfig.colors
+			config: streamerConfig,
 		});
 	} else {
 		var validStates = JSON.stringify((await db.query(queries.getValidStates).then((result: pg.QueryResult) => result.rows)).map((row: any) => row.state));
@@ -452,10 +544,9 @@ app.get("/", async (request, response) => {
 			userProfilePicture: request.session.user.profile_image_url,
 			validStates: validStates,
 			isStreamer: streamerInfo['userid'] == request.session.user.id,
-			pageTitle: streamerConfig.title,
 			streamerName: streamerInfo['displayname'],
 			streamerProfilePicture: streamerInfo['imageurl'],
-			colors: streamerConfig.colors
+			config: streamerConfig,
 		});
 	}
 });

src/cronjobs/index.ts

@@ -1,7 +1,11 @@
 import { processBans } from './processBans';
+import { processFollows } from './processFollows';
+import { processSubscriptions } from './processSubscriptions';
 import { processEmptyMetadata } from './processEmptyMetadata';
 
 export = {
 	processBans,
+	processFollows,
+	processSubscriptions,
 	processEmptyMetadata
 }

src/cronjobs/processBans.ts

@@ -7,13 +7,13 @@ export async function processBans(streamer: twitch.StreamerUserIdTokenPair) {
 	try {
 		await dbconn.query('BEGIN');
 		await dbconn.query("DELETE FROM bans");
-		//log(LogLevel.DEBUG,"Ban API response:")
-		//log(LogLevel.DEBUG,JSON.stringify(response,null,2));
 		var response = await twitch.streamerApiRequest(streamer,
 			`/moderation/banned?broadcaster_id=${streamer.userid}&first=100`);
+		log(LogLevel.DEBUG,"Ban API response:")
+		log(LogLevel.DEBUG,JSON.stringify(response,null,2));
 		while (true) {
 			var insertBanQuery = "INSERT INTO bans (userid) VALUES ";
-			var banRow = 0;
+			var banRow = 0; // Used for $1, $2, etc. in parameterized query
 			var bansArray: number[] = [];
 			if (Object.keys(response.data).length > 0) {
 				for (var ban of response.data) {
@@ -37,11 +37,11 @@ export async function processBans(streamer: twitch.StreamerUserIdTokenPair) {
 				var oldFirstUserid = response.data[0].user_id;
 				response = await twitch.streamerApiRequest(streamer,
 					`/moderation/banned?broadcaster_id=${streamer.userid}&after=${response.pagination.cursor}&first=100`);
+				log(LogLevel.DEBUG,"Ban API response:");
+				log(LogLevel.DEBUG,JSON.stringify(response,null,2));
 				// Work around broken api endpoint giving a cursor referring to the
 				// current page, causing an infinite loop
 				if (oldFirstUserid == response.data[0].user_id) break;
-				//log(LogLevel.DEBUG,"Ban API response:");
-				//log(LogLevel.DEBUG,JSON.stringify(response,null,2));
 			} else {
 				break;
 			}

src/cronjobs/processFollows.ts

@@ -0,0 +1,53 @@
+import * as twitch from "../twitch";
+import { log, LogLevel } from "../logging"
+import db from "../db";
+
+export async function processFollows(streamer: twitch.StreamerUserIdTokenPair) {
+	var dbconn = await db.connect();
+	try {
+		await dbconn.query('BEGIN');
+		await dbconn.query("DELETE FROM follows");
+		// Insert the streamer as a follower
+		await dbconn.query({text: "INSERT INTO follows (userid) VALUES ($1)", values: [streamer.userid]})
+		var response = await twitch.streamerApiRequest(streamer,
+			`/users/follows?to_id=${streamer.userid}&first=100`);
+		log(LogLevel.DEBUG,"Follows API response:")
+		log(LogLevel.DEBUG,JSON.stringify(response,null,2));
+		while (true) {
+			var insertFollowQuery = "INSERT INTO follows (userid) VALUES ";
+			var followRow = 0; // Used for $1, $2, etc. in parameterized query
+			var followsArray: number[] = [];
+			if (Object.keys(response.data).length > 0) {
+				for (var follow of response.data) {
+					followRow++;
+					insertFollowQuery += `($${followRow}), `;
+					followsArray.push(follow.from_id as number);
+				}
+				insertFollowQuery = insertFollowQuery.slice(0,-2); // Cut last `, ` off of the end
+				var followQueryConfig = {
+					text: insertFollowQuery,
+					values: followsArray
+				};
+				log(LogLevel.DEBUG,"followQueryConfig object:")
+				log(LogLevel.DEBUG,JSON.stringify(followQueryConfig,null,2))
+				await dbconn.query(followQueryConfig);
+			}
+			if (response.pagination.cursor) {
+				response = await twitch.streamerApiRequest(streamer,
+					`/users/follows?to_id=${streamer.userid}&after=${response.pagination.cursor}&first=100`);
+				log(LogLevel.DEBUG,"Follow API response:");
+				log(LogLevel.DEBUG,JSON.stringify(response,null,2));
+			} else {
+				break;
+			}
+		}
+		await dbconn.query("CALL update_scores()");
+		await dbconn.query('COMMIT');
+	} catch (e) {
+		log(LogLevel.ERROR,"cronjobs.processFollows: Exception thrown; rolling back");
+		await dbconn.query('ROLLBACK');
+		throw(e);
+	} finally {
+		await dbconn.release();
+	}
+}

src/cronjobs/processSubscriptions.ts

@@ -0,0 +1,55 @@
+import * as twitch from "../twitch";
+import { log, LogLevel } from "../logging"
+import db from "../db";
+
+export async function processSubscriptions(streamer: twitch.StreamerUserIdTokenPair) {
+	var dbconn = await db.connect();
+	try {
+		await dbconn.query('BEGIN');
+		await dbconn.query("DELETE FROM subscriptions");
+		var response = await twitch.streamerApiRequest(streamer,
+			`/subscriptions?broadcaster_id=${streamer.userid}&first=100`);
+		log(LogLevel.DEBUG,"Subscriptions API response:")
+		log(LogLevel.DEBUG,JSON.stringify(response,null,2));
+		while (true) {
+			var insertSubscriptionQuery = "INSERT INTO subscriptions (userid) VALUES ";
+			var subscriptionRow = 0; // Used for $1, $2, etc. in parameterized query
+			var subscriptionsArray: number[] = [];
+			if (Object.keys(response.data).length > 0) {
+				for (var subscription of response.data) {
+					subscriptionRow++;
+					insertSubscriptionQuery += `($${subscriptionRow}), `;
+					subscriptionsArray.push(subscription.user_id as number);
+				}
+				insertSubscriptionQuery = insertSubscriptionQuery.slice(0,-2); // Cut last `, ` off of the end
+				insertSubscriptionQuery += " ON CONFLICT DO NOTHING"; // Deal with broken endpoint returning dupes
+				var subscriptionQueryConfig = {
+					text: insertSubscriptionQuery,
+					values: subscriptionsArray
+				};
+				log(LogLevel.DEBUG,"subscriptionQueryConfig object:")
+				log(LogLevel.DEBUG,JSON.stringify(subscriptionQueryConfig,null,2))
+				await dbconn.query(subscriptionQueryConfig);
+			}
+			if (response.pagination.cursor) {
+				var oldFirstUserid = response.data[0].user_id;
+				response = await twitch.streamerApiRequest(streamer,
+					`/subscriptions?broadcaster_id=${streamer.userid}&after=${response.pagination.cursor}&first=100`);
+				log(LogLevel.DEBUG,"Subscription API response:");
+				log(LogLevel.DEBUG,JSON.stringify(response,null,2));
+				if (response.data.length === 0 || oldFirstUserid === response.data[0].user_id) break;
+			} else {
+				break;
+			}
+		}
+		await dbconn.query("CALL update_scores()");
+		await dbconn.query('COMMIT');
+	} catch (e) {
+		log(LogLevel.ERROR,"cronjobs.processSubscriptions: Exception: " + e);
+		log(LogLevel.ERROR,"cronjobs.processSubscriptions: Exception thrown; rolling back");
+		await dbconn.query('ROLLBACK');
+		throw(e);
+	} finally {
+		await dbconn.release();
+	}
+}

src/queries.ts

@@ -47,34 +47,25 @@ export const updateColors = {
 	text: "UPDATE config SET colors = $1"
 }
 
-// Request-related queries
-export const getRequests = {
-	name: "getRequests",
-	text: "SELECT * FROM requests_vw \
-		JOIN states ON requests_vw.state = states.state \
-		WHERE active LIMIT $1"
+export const updateVotePoints = {
+	name: "updateVotePoints",
+	text: "CALL update_vote_points($1,$2,$3)"
 }
 
-export const getRequestsVoted = {
-	name: "getRequestsVoted",
-	text: "SELECT * FROM get_requests_voted($2) \
-		JOIN states ON get_requests_voted.state = states.state \
-		WHERE active LIMIT $1"
+export const getRequestsTotal = {
+	name: "getRequestsTotal",
+	text: "SELECT COUNT(*) FROM requests_vw \
+		JOIN states ON requests_vw.state = states.state WHERE active"
 }
 
-export const getAllRequests = {
-	name: "getAllRequests",
-	text: "SELECT * FROM requests_vw LIMIT $1"
-}
-
-export const getAllRequestsVoted = {
-	name: "getAllRequestsVoted",
-	text: "SELECT * FROM get_requests_voted($2) LIMIT $1"
+export const getAllRequestsTotal = {
+	name: "getAllRequestsTotal",
+	text: "SELECT COUNT(*) FROM requests_vw"
 }
 
 export const checkRequestExists = {
 	name: "checkRequestExists",
- 	text: "SELECT url FROM requests WHERE url = $1"
+ 	text: "SELECT url,requester,state FROM requests_vw WHERE url = $1"
 }
 
 export const addRequest = {
@@ -132,10 +123,15 @@ export const getAndLockCronJob = {
 	text: `SELECT * FROM cron
            WHERE (lastSuccess + runInterval) < now()
            AND jobName = $1
-           FOR UPDATE SKIP LOCKED;`
+           FOR UPDATE SKIP LOCKED`
 }
 
 export const updateCronJobLastSuccess = {
 	name: "updateCronJobLastSuccess",
 	text: "UPDATE cron SET lastSuccess = now() WHERE jobName = $1"
 }
+
+export const getRateLimitStatus = {
+	name: "getRateLimitStatus",
+	text: "SELECT * FROM ratelimit_vw WHERE userid = $1"
+}

src/requests.ts

@@ -4,30 +4,58 @@ import { log, LogLevel } from "./logging"
 import pg from "pg";
 import db from "./db";
 
-export async function getRequests(count: number) {
-	var query = Object.assign(queries.getRequests, { values: [count] });
+export async function getRequests(count: number, offset: number, sort: string) {
+	var query = {
+		text: "SELECT * FROM requests_vw \
+			JOIN states ON requests_vw.state = states.state WHERE active \
+			ORDER BY " + sort + " LIMIT $1 OFFSET $2",
+		values: [count,offset]
+	};
 	return db.query(query)
 	.then((result: pg.QueryResult) => result.rows);
 };
 
-export async function getAllRequests(count: number) {
-	var query = Object.assign(queries.getAllRequests, { values: [count] });
+export async function getRequestsVoted(count: number, offset: number, sort: string, user: number) {
+	var query = {
+		text: "SELECT * FROM get_requests_voted($3) \
+			JOIN states ON get_requests_voted.state = states.state WHERE active \
+			ORDER BY " + sort + " LIMIT $1 OFFSET $2",
+		values: [count,offset,user]
+	};
 	return db.query(query)
 	.then((result: pg.QueryResult) => result.rows);
 };
 
-export async function getRequestsVoted(count: number, user: number) {
-	var query = Object.assign(queries.getRequestsVoted, { values: [count,user] });
+export async function getRequestsTotal() {
+	return db.query(queries.getRequestsTotal)
+	.then((result: pg.QueryResult) => result.rows[0]["count"]);
+};
+
+export async function getAllRequests(count: number, offset: number, sort: string) {
+	var query = {
+		text: "SELECT * FROM requests_vw \
+			ORDER BY " + sort + " LIMIT $1 OFFSET $2",
+		values: [count,offset]
+	};
 	return db.query(query)
 	.then((result: pg.QueryResult) => result.rows);
 };
 
-export async function getAllRequestsVoted(count: number,user: number) {
-	var query = Object.assign(queries.getAllRequestsVoted, { values: [count,user] });
+export async function getAllRequestsVoted(count: number, offset: number, sort: string, user: number) {
+	var query = {
+		text: "SELECT * FROM get_requests_voted($3) \
+			ORDER BY " + sort + " LIMIT $1 OFFSET $2",
+		values: [count,offset,user]
+	};
 	return db.query(query)
 	.then((result: pg.QueryResult) => result.rows);
 };
 
+export async function getAllRequestsTotal() {
+	return db.query(queries.getAllRequestsTotal)
+	.then((result: pg.QueryResult) => result.rows[0]["count"]);
+};
+
 const validUrlRegexes = [
 	/^https:\/\/www\.youtube\.com\/watch\?v=[a-zA-Z0-9_-]{11}$/
 ];
@@ -59,6 +87,7 @@ async function retrieveYoutubeMetadata(url: string) {
 }
 
 export async function addRequest(url: string, requester: string): Promise<[number,string]> {
+	// Check that URL is of an accepted format
 	var validUrl = false;
 	for (var regex of validUrlRegexes) {
 		if (regex.test(url)) {
@@ -67,10 +96,24 @@ export async function addRequest(url: string, requester: string): Promise<[numbe
 		}
 	}
 	if (!validUrl) return [400, "Invalid song URL."];
-	var result = await checkRequestExists(url)
-	if (result) {
-		return [200,`Song already requested by ${result.rows[0].requester}. State: ${result.rows[0].state}`]
+
+	// Check whether the URL has already been requested
+	var existsResult = await checkRequestExists(url)
+	if (existsResult) {
+		console.log(existsResult);
+		return [200,`Song already requested by ${existsResult.rows[0].requester}. State: ${existsResult.rows[0].state}`]
 	}
+
+	// Check whether the user has hit their rate limit
+	var rateLimitQuery = Object.assign(queries.getRateLimitStatus, { values: [requester] });
+	var rateLimitResult = (await db.query(rateLimitQuery)).rows[0];
+	if (rateLimitResult.status) {
+		return [429,`You have reached your maximum of ${rateLimitResult.max} requests per day.
+			Please try again later.\n
+			Tip: Removing one of your requests from the past 24 hours by retracting your vote will allow you to replace it with another.`];
+	}
+
+	// Add the request
 	var query = Object.assign(queries.addRequest, { values: [url,requester] });
 	return db.query(query)
 		.then(async () => {

src/twitch.ts

@@ -53,6 +53,7 @@ export async function apiRequest(tokens: TokenPair, endpoint: string): Promise <
 	return fetch("https://api.twitch.tv/helix" + endpoint, { headers: headers })
 		.then(async (res: FetchResponse) => {
 			if (res.status == 200) {
+				log(LogLevel.DEBUG,"twitch.apiRequest: Request returned 200 for " + endpoint);
 				return res.json();
 			} else {
 				log(LogLevel.WARNING,"twitch.apiRequest: Failed API request (pre-refresh):");

src/version.ts

@@ -5,7 +5,7 @@ import pg from "pg";
 import db from "./db";
 
 var versionMajor = 0;
-var versionMinor = 4;