From af97647033f498b14011a65b69b4a6a9cf9264e3 Mon Sep 17 00:00:00 2001 From: Dessa Simpson Date: Sun, 29 Nov 2020 01:17:58 -0700 Subject: [PATCH] Make processBans support pagination --- src/cronjobs/processBans.ts | 58 ++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/src/cronjobs/processBans.ts b/src/cronjobs/processBans.ts index 1fe84ab..d21cb76 100644 --- a/src/cronjobs/processBans.ts +++ b/src/cronjobs/processBans.ts @@ -3,31 +3,49 @@ import { log, LogLevel } from "../logging" import db from "../db"; export async function processBans(streamer: twitch.StreamerUserIdTokenPair) { - var response = await twitch.streamerApiRequest(streamer, - "/moderation/banned?broadcaster_id=" + streamer.userid); - var dbconn = await db.connect(); + var dbconn = await db.connect(); try { await dbconn.query('BEGIN'); await dbconn.query("DELETE FROM bans"); - log(LogLevel.DEBUG,"Ban list:") - log(LogLevel.DEBUG,JSON.stringify(response.data,null,2)); - var insertBanQuery = "INSERT INTO bans (userid) VALUES "; - var banRow = 0; - var bansArray: number[] = []; - for (var ban of response.data) { - if (ban.expires_at == '') { - banRow++; - insertBanQuery += `($${banRow}), `; - bansArray.push(ban.user_id as number); + //log(LogLevel.DEBUG,"Ban API response:") + //log(LogLevel.DEBUG,JSON.stringify(response,null,2)); + var response = await twitch.streamerApiRequest(streamer, + `/moderation/banned?broadcaster_id=${streamer.userid}&first=100`); + while (true) { + var insertBanQuery = "INSERT INTO bans (userid) VALUES "; + var banRow = 0; + var bansArray: number[] = []; + if (Object.keys(response.data).length > 0) { + for (var ban of response.data) { + if (ban.expires_at == '') { + banRow++; + insertBanQuery += `($${banRow}), `; + bansArray.push(ban.user_id as number); + } + } + insertBanQuery = insertBanQuery.slice(0,-2); // Cut last `, ` off of the end + insertBanQuery += " ON CONFLICT DO NOTHING"; // Deal with broken endpoint returning dupes + var banQueryConfig = { + text: insertBanQuery, + values: bansArray + }; + log(LogLevel.DEBUG,"banQueryConfig object:") + log(LogLevel.DEBUG,JSON.stringify(banQueryConfig,null,2)) + await dbconn.query(banQueryConfig); + } + if (response.pagination.cursor) { + var oldFirstUserid = response.data[0].user_id; + response = await twitch.streamerApiRequest(streamer, + `/moderation/banned?broadcaster_id=${streamer.userid}&after=${response.pagination.cursor}&first=100`); + // Work around broken api endpoint giving a cursor referring to the + // current page, causing an infinite loop + if (oldFirstUserid == response.data[0].user_id) break; + //log(LogLevel.DEBUG,"Ban API response:"); + //log(LogLevel.DEBUG,JSON.stringify(response,null,2)); + } else { + break; } } - var banQueryConfig = { - text: insertBanQuery.slice(0,-2), // Cut last `, ` off of the end - values: bansArray - }; - log(LogLevel.DEBUG,"banQueryConfig object:") - log(LogLevel.DEBUG,JSON.stringify(banQueryConfig,null,2)) - await dbconn.query(banQueryConfig); await dbconn.query("CALL update_scores()"); await dbconn.query('COMMIT'); } catch (e) {