From af97647033f498b14011a65b69b4a6a9cf9264e3 Mon Sep 17 00:00:00 2001
From: Dessa Simpson <dxs@k7dxs.net>
Date: Sun, 29 Nov 2020 01:17:58 -0700
Subject: [PATCH] Make processBans support pagination

---
 src/cronjobs/processBans.ts | 58 ++++++++++++++++++++++++-------------
 1 file changed, 38 insertions(+), 20 deletions(-)

diff --git a/src/cronjobs/processBans.ts b/src/cronjobs/processBans.ts
index 1fe84ab..d21cb76 100644
--- a/src/cronjobs/processBans.ts
+++ b/src/cronjobs/processBans.ts
@@ -3,31 +3,49 @@ import { log, LogLevel } from "../logging"
 import db from "../db";
 
 export async function processBans(streamer: twitch.StreamerUserIdTokenPair) {
-	var response = await twitch.streamerApiRequest(streamer,
-		"/moderation/banned?broadcaster_id=" + streamer.userid);
-	var dbconn = await db.connect();
+		var dbconn = await db.connect();
 	try {
 		await dbconn.query('BEGIN');
 		await dbconn.query("DELETE FROM bans");
-		log(LogLevel.DEBUG,"Ban list:")
-		log(LogLevel.DEBUG,JSON.stringify(response.data,null,2));
-		var insertBanQuery = "INSERT INTO bans (userid) VALUES ";
-		var banRow = 0;
-		var bansArray: number[] = [];
-		for (var ban of response.data) {
-			if (ban.expires_at == '') {
-				banRow++;
-				insertBanQuery += `($${banRow}), `;
-				bansArray.push(ban.user_id as number);
+		//log(LogLevel.DEBUG,"Ban API response:")
+		//log(LogLevel.DEBUG,JSON.stringify(response,null,2));
+		var response = await twitch.streamerApiRequest(streamer,
+			`/moderation/banned?broadcaster_id=${streamer.userid}&first=100`);
+		while (true) {
+			var insertBanQuery = "INSERT INTO bans (userid) VALUES ";
+			var banRow = 0;
+			var bansArray: number[] = [];
+			if (Object.keys(response.data).length > 0) {
+				for (var ban of response.data) {
+					if (ban.expires_at == '') {
+						banRow++;
+						insertBanQuery += `($${banRow}), `;
+						bansArray.push(ban.user_id as number);
+					}
+				}
+				insertBanQuery = insertBanQuery.slice(0,-2); // Cut last `, ` off of the end
+				insertBanQuery += " ON CONFLICT DO NOTHING"; // Deal with broken endpoint returning dupes
+				var banQueryConfig = {
+					text: insertBanQuery,
+					values: bansArray
+				};
+				log(LogLevel.DEBUG,"banQueryConfig object:")
+				log(LogLevel.DEBUG,JSON.stringify(banQueryConfig,null,2))
+				await dbconn.query(banQueryConfig);
+			}
+			if (response.pagination.cursor) {
+				var oldFirstUserid = response.data[0].user_id;
+				response = await twitch.streamerApiRequest(streamer,
+					`/moderation/banned?broadcaster_id=${streamer.userid}&after=${response.pagination.cursor}&first=100`);
+				// Work around broken api endpoint giving a cursor referring to the
+				// current page, causing an infinite loop
+				if (oldFirstUserid == response.data[0].user_id) break;
+				//log(LogLevel.DEBUG,"Ban API response:");
+				//log(LogLevel.DEBUG,JSON.stringify(response,null,2));
+			} else {
+				break;
 			}
 		}
-		var banQueryConfig = {
-			text: insertBanQuery.slice(0,-2), // Cut last `, ` off of the end
-			values: bansArray
-		};
-		log(LogLevel.DEBUG,"banQueryConfig object:")
-		log(LogLevel.DEBUG,JSON.stringify(banQueryConfig,null,2))
-		await dbconn.query(banQueryConfig);
 		await dbconn.query("CALL update_scores()");
 		await dbconn.query('COMMIT');
 	} catch (e) {