diff --git a/.dockerignore b/.dockerignore index d95cec9..fe9214f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,7 @@ Dockerfile +Dockerfile.prod +docker-compose.yml .dockerignore .git .gitignore +node_modules diff --git a/Dockerfile.prod b/Dockerfile.prod new file mode 100644 index 0000000..49539fd --- /dev/null +++ b/Dockerfile.prod @@ -0,0 +1,24 @@ +FROM node:14 AS builder + +USER node + +ENV PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/app/node_modules/.bin" + +COPY --chown=node . /app +WORKDIR /app +RUN ["npm", "install"] +RUN ["tsc", "--outDir", "build"] + +FROM node:14-alpine + +EXPOSE 3000 +ENV PORT 3000 +ENV PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/app/node_modules/.bin" NODE_ENV="production" + +USER node +WORKDIR /app +COPY --from=builder /app/build build +COPY --from=builder /app/node_modules node_modules +COPY --from=builder /app/public public +COPY --from=builder /app/views views +CMD ["node", "./build/app.js"] diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..c60d7c9 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,30 @@ +# Release Checklist + +## Release Type +- Were there any database schema changes? +- Are there any significant UI changes? +- Have any significant new features been added? +- Are there any API changes? + +If the answer to any of the above is yes, then the release MUST be a major or minor release. Otherwise, the release MAY be a patch release (at developer's discretion). + +## Major/Minor Releases +- [ ] Add a commit which adds a database upgrade script at `db/upgrade/[oldversion]-[newversion].sql` + - ALWAYS use a transaction for the entirety of this file + - At minimum, DB version must be bumped + - Test the upgrade script as follows: + ``` + git checkout [previous release tag] + docker-compose down + docker-compose up + psql -h 0 -U postgres < db/upgrade/v[previous]-v[current].sql + ``` + Update the version in `src/version.ts` and verify the app works as expected. +- [ ] Add a commit which bumps the version in `src/version.ts` and `db/00-version.sql`, entitled `Bump version to vMAJOR.MINOR` +- [ ] Tag the latest commit with `vMAJOR.MINOR` +- [ ] Write release notes + +## Patch Releases +- [ ] Add a commit which bumps the patch level in `src/version.ts`, entitled `Bump version to vMAJOR.MINOR.PATCH` +- [ ] Tag the latest commit with `vMAJOR.MINOR.PATCH` +- [ ] Write release notes \ No newline at end of file diff --git a/db/00-version.sql b/db/00-version.sql index 5c43aec..6686897 100644 --- a/db/00-version.sql +++ b/db/00-version.sql @@ -8,4 +8,4 @@ CREATE OR REPLACE FUNCTION get_version() RETURNS VARCHAR AS $$SELECT major || '.' || minor FROM version $$ LANGUAGE SQL; -INSERT INTO version (major,minor) VALUES (0,2); +INSERT INTO version (major,minor) VALUES (0,8); diff --git a/db/05-config.sql b/db/05-config.sql index edf6d9e..2e830ab 100644 --- a/db/05-config.sql +++ b/db/05-config.sql @@ -1,10 +1,14 @@ CREATE TABLE config ( rowlock bool DEFAULT TRUE UNIQUE NOT NULL CHECK (rowlock = TRUE), - normaluservotepoints int NOT NULL, - followervotepoints int NOT NULL, - subscribervotepoints int NOT NULL, + normaluservotepoints int NOT NULL DEFAULT 10, + followervotepoints int NOT NULL DEFAULT 50, + subscribervotepoints int NOT NULL DEFAULT 100, + normaluserratelimit int NOT NULL DEFAULT 1, + followerratelimit int NOT NULL DEFAULT 2, + subscriberratelimit int NOT NULL DEFAULT 3, + title varchar NOT NULL DEFAULT '{username}''s Learn Request Queue', + colors jsonb NOT NULL DEFAULT '{"bg": {"primary": "#444444","table": "#282828","navbar": "#666666","error": "#ff0000"},"fg": {"primary": "#dddddd","ahover": "#ffffff","title": "#eeeeee"}}', PRIMARY KEY (rowLock) ); -INSERT INTO config (normalUserVotePoints,followerVotePoints,subscriberVotePoints) - VALUES (10,50,100); +INSERT INTO config (rowlock) VALUES (true); diff --git a/db/10-cron.sql b/db/10-cron.sql new file mode 100644 index 0000000..a0b1941 --- /dev/null +++ b/db/10-cron.sql @@ -0,0 +1,13 @@ +CREATE TABLE cron ( + jobName varchar UNIQUE NOT NULL, -- Application-recognizable name for the job + runinterval interval NOT NULL, -- Duration between runs + -- Last successful run - only gets updated if run is successful + lastSuccess timestamptz DEFAULT to_timestamp(0), -- Defaults to beginning of time + PRIMARY KEY(jobName) +); + +INSERT INTO cron (jobName,runInterval) VALUES +('processBans','30 minutes'), +('processFollows','30 minutes'), +('processSubscriptions','30 minutes'), +('processEmptyMetadata','1 day'); diff --git a/db/10-users.sql b/db/10-users.sql index dc0aef7..7774265 100644 --- a/db/10-users.sql +++ b/db/10-users.sql @@ -2,7 +2,5 @@ CREATE TABLE users ( userId int NOT NULL, displayName varchar NOT NULL, imageUrl varchar, - isFollower boolean NOT NULL DEFAULT FALSE, - isSubscriber boolean NOT NULL DEFAULT FALSE, PRIMARY KEY (userId) ); diff --git a/db/15-streamer.sql b/db/15-streamer.sql index 53068c8..b39604c 100644 --- a/db/15-streamer.sql +++ b/db/15-streamer.sql @@ -3,4 +3,4 @@ CREATE TABLE streamer ( tokenPair json, PRIMARY KEY (userid), FOREIGN KEY (userid) REFERENCES users(userid) -) +); diff --git a/db/20-requests.sql b/db/20-requests.sql index c3f3cf5..b63c359 100644 --- a/db/20-requests.sql +++ b/db/20-requests.sql @@ -1,5 +1,5 @@ CREATE TABLE requests ( - url varchar NOT NULL UNIQUE, + url varchar NOT NULL, requester int NOT NULL, state varchar NOT NULL DEFAULT 'Requested', reqTimestamp timestamptz NOT NULL DEFAULT NOW(), diff --git a/db/30-votes.sql b/db/30-votes.sql index 0b878ba..8da76f0 100644 --- a/db/30-votes.sql +++ b/db/30-votes.sql @@ -1,6 +1,6 @@ CREATE TABLE votes ( - requestUrl varchar, - userId int, + requestUrl varchar NOT NULL, + userId int NOT NULL, PRIMARY KEY (requestUrl,userId), FOREIGN KEY (requestUrl) REFERENCES requests(url) ON DELETE CASCADE, FOREIGN KEY (userId) REFERENCES users(userId) ON DELETE CASCADE diff --git a/db/50-bans.sql b/db/50-bans.sql index 979ae21..20ecfa1 100644 --- a/db/50-bans.sql +++ b/db/50-bans.sql @@ -1,4 +1,4 @@ CREATE TABLE bans ( userid integer, PRIMARY KEY (userid) -) +); diff --git a/db/50-follows.sql b/db/50-follows.sql new file mode 100644 index 0000000..63e1200 --- /dev/null +++ b/db/50-follows.sql @@ -0,0 +1,4 @@ +CREATE TABLE follows ( + userid integer, + PRIMARY KEY (userid) +); diff --git a/db/50-subscriptions.sql b/db/50-subscriptions.sql new file mode 100644 index 0000000..7e23cf1 --- /dev/null +++ b/db/50-subscriptions.sql @@ -0,0 +1,4 @@ +CREATE TABLE subscriptions ( + userid integer, + PRIMARY KEY (userid) +); diff --git a/db/90-views.sql b/db/90-views.sql index e274069..a31ec16 100644 --- a/db/90-views.sql +++ b/db/90-views.sql @@ -2,8 +2,7 @@ CREATE OR REPLACE VIEW requests_vw AS SELECT url,COALESCE(videoTitle,url) AS title,displayName AS requester,imageUrl,state,score,reqTimestamp FROM requests JOIN requestMetadata USING (url) JOIN scores USING (url) - JOIN users ON requests.requester = users.userid - ORDER BY score DESC, reqTimestamp ASC; + JOIN users ON requests.requester = users.userid; CREATE OR REPLACE FUNCTION get_requests_voted(votinguserid int) RETURNS TABLE ( @@ -21,8 +20,7 @@ CREATE OR REPLACE FUNCTION get_requests_voted(votinguserid int) SELECT url,title,requester,imageUrl,state,score,reqTimestamp, (CASE WHEN votes.userid IS NULL THEN FALSE ELSE TRUE END) AS voted FROM requests_vw - LEFT JOIN votes ON (requests_vw.url = votes.requesturl AND votes.userid = votinguserid) - ORDER BY score DESC, reqTimestamp ASC; + LEFT JOIN votes ON (requests_vw.url = votes.requesturl AND votes.userid = votinguserid); $$; /* @@ -40,18 +38,20 @@ CREATE OR REPLACE VIEW vote_score_vw AS COUNT(votes.requesturl) AS count, COALESCE( SUM(CASE - WHEN users.isfollower = FALSE AND users.issubscriber = FALSE + WHEN follows.userid IS NULL AND subscriptions.userid IS NULL + AND votes.userid IS NOT NULL THEN votepoints.normaluservotepoints - WHEN users.isfollower = TRUE AND users.issubscriber = FALSE + WHEN follows.userid IS NOT NULL AND subscriptions.userid IS NULL THEN votepoints.followervotepoints - WHEN users.issubscriber = TRUE + WHEN subscriptions.userid IS NOT NULL THEN votepoints.subscribervotepoints END), 0 ) AS votescore FROM requests LEFT JOIN votes ON votes.requesturl = requests.url - LEFT JOIN users on votes.userid = users.userid - LEFT JOIN bans ON users.userid = bans.userid + LEFT JOIN bans ON votes.userid = bans.userid + LEFT JOIN follows ON votes.userid = follows.userid + LEFT JOIN subscriptions ON votes.userid = subscriptions.userid CROSS JOIN votepoints WHERE bans.userid IS NULL GROUP BY url; @@ -60,3 +60,22 @@ CREATE OR REPLACE VIEW streamer_user_vw AS SELECT users.userid as userid, users.displayname as displayname, users.imageurl as imageurl FROM streamer LEFT JOIN users ON streamer.userid = users.userid; + +CREATE OR REPLACE VIEW ratelimit_vw AS + SELECT users.userid,COALESCE(count,0),ratelimit.reqcount AS max,COALESCE(count,0) >= ratelimit.reqcount AS status + FROM users + LEFT JOIN (SELECT requester,COUNT(url) + FROM requests + WHERE reqtimestamp > (now() - '24 hours'::interval) + GROUP BY requests.requester + ) AS requests ON users.userid = requests.requester + LEFT JOIN follows ON requests.requester = follows.userid + LEFT JOIN subscriptions ON requests.requester = subscriptions.userid + CROSS JOIN config + CROSS JOIN LATERAL (VALUES ( + CASE + WHEN follows.userid IS NULL AND subscriptions.userid IS NULL THEN config.normaluserratelimit + WHEN follows.userid IS NOT NULL AND subscriptions.userid IS NULL THEN config.followerratelimit + WHEN subscriptions.userid IS NOT NULL THEN config.subscriberratelimit + END + )) AS ratelimit(reqcount); diff --git a/db/95-procedures.sql b/db/95-procedures.sql index f2427bf..650f119 100644 --- a/db/95-procedures.sql +++ b/db/95-procedures.sql @@ -16,6 +16,13 @@ CREATE OR REPLACE PROCEDURE add_request(url varchar,requester int) CALL update_scores(); $$; +CREATE OR REPLACE PROCEDURE clear_zero_votes() + LANGUAGE SQL + AS $$ + DELETE FROM requests WHERE NOT EXISTS + (SELECT FROM votes WHERE requests.url = votes.requesturl); + $$; + CREATE OR REPLACE PROCEDURE add_vote(url varchar,voteuser int) LANGUAGE SQL AS $$ @@ -28,6 +35,7 @@ CREATE OR REPLACE PROCEDURE delete_vote(url varchar,voteuser int) AS $$ DELETE FROM votes WHERE requesturl = url AND userid = voteuser; CALL update_scores(); + CALL clear_zero_votes(); $$; CREATE OR REPLACE PROCEDURE update_request_score_modifier(updateurl varchar, scoreDiff int) @@ -36,3 +44,11 @@ CREATE OR REPLACE PROCEDURE update_request_score_modifier(updateurl varchar, sco UPDATE scores SET scoreModifier = scoreModifier + scoreDiff WHERE url = updateurl; CALL update_scores(); $$; + +CREATE OR REPLACE PROCEDURE update_vote_points(normaluser int, follower int, subscriber int) + LANGUAGE SQL + AS $$ + UPDATE config SET normaluservotepoints = normaluser, + followervotepoints = follower, subscribervotepoints = subscriber; + CALL update_scores(); + $$; diff --git a/db/testdata.sql b/db/testdata.sql index 83428e6..98db264 100644 --- a/db/testdata.sql +++ b/db/testdata.sql @@ -1,8 +1,16 @@ -INSERT INTO users (userid,displayName,isFollower,isSubscriber) VALUES - (001,'TestUser',false,false), - (002,'TestFollower',true,false), - (003,'TestSubscriber',true,true), - (004,'TestSubNonFollower',false,true); +INSERT INTO users (userid,displayName) VALUES + (001,'TestUser'), + (002,'TestFollower'), + (003,'TestSubscriber'), + (004,'TestSubNonFollower'); + +INSERT INTO follows (userid) VALUES + (002), + (003); + +INSERT INTO subscriptions (userid) VALUES + (003), + (004); CALL add_request('https://www.youtube.com/watch?v=dQw4w9WgXcQ',001); CALL add_request('https://www.youtube.com/watch?v=C5oeWHngDS4',002); diff --git a/db/upgrade/v0.2-v0.3.sql b/db/upgrade/v0.2-v0.3.sql new file mode 100644 index 0000000..defe309 --- /dev/null +++ b/db/upgrade/v0.2-v0.3.sql @@ -0,0 +1,18 @@ +BEGIN; + +UPDATE version SET minor = 3; + +CREATE TABLE cron ( + jobName varchar UNIQUE NOT NULL, -- Application-recognizable name for the job + runinterval interval NOT NULL, -- Duration between runs + -- Last successful run - only gets updated if run is successful + lastSuccess timestamptz DEFAULT to_timestamp(0), -- Defaults to beginning of time + PRIMARY KEY(jobName) +); + +INSERT INTO cron (jobName,runInterval) VALUES +('processBans','30 minutes'), +('processEmptyMetadata','1 day'); + + +COMMIT; diff --git a/db/upgrade/v0.3-0.4.sql b/db/upgrade/v0.3-0.4.sql new file mode 100644 index 0000000..e15ef43 --- /dev/null +++ b/db/upgrade/v0.3-0.4.sql @@ -0,0 +1,9 @@ +BEGIN; + +UPDATE version SET minor = 4; + +ALTER TABLE config ADD COLUMN title varchar, ADD COLUMN colors jsonb; +UPDATE config SET title = '{username}''s Learn Request Queue', colors = '{"bg": {"primary": "#444444","table": "#282828","navbar" : "#666666","error": "#ff0000"},"fg": {"primary": "#dddddd","ahover": "#ffffff","title": "#eeeeee"}}'; +ALTER TABLE config ALTER COLUMN title SET NOT NULL, ALTER COLUMN colors SET NOT NULL; + +COMMIT; diff --git a/db/upgrade/v0.4-0.5.sql b/db/upgrade/v0.4-0.5.sql new file mode 100644 index 0000000..df6d8e7 --- /dev/null +++ b/db/upgrade/v0.4-0.5.sql @@ -0,0 +1,53 @@ +BEGIN; + +UPDATE version SET minor = 5; + +CREATE OR REPLACE PROCEDURE update_vote_points(normaluser int, follower int, subscriber int) + LANGUAGE SQL + AS $$ + UPDATE config SET normaluservotepoints = normaluser, + followervotepoints = follower, subscribervotepoints = subscriber; + CALL update_scores(); + $$; + +CREATE TABLE follows ( + userid integer, + PRIMARY KEY (userid) +); + +CREATE TABLE subscriptions ( + userid integer, + PRIMARY KEY (userid) +); + +CREATE OR REPLACE VIEW vote_score_vw AS + WITH votepoints AS (SELECT normaluservotepoints, followervotepoints, subscribervotepoints FROM config) + SELECT requests.url AS url, + COUNT(votes.requesturl) AS count, + COALESCE( + SUM(CASE + WHEN follows.userid IS NULL AND subscriptions.userid IS NULL + AND votes.userid IS NOT NULL + THEN votepoints.normaluservotepoints + WHEN follows.userid IS NOT NULL AND subscriptions.userid IS NULL + THEN votepoints.followervotepoints + WHEN subscriptions.userid IS NOT NULL + THEN votepoints.subscribervotepoints + END), 0 + ) AS votescore + FROM requests + LEFT JOIN votes ON votes.requesturl = requests.url + LEFT JOIN bans ON votes.userid = bans.userid + LEFT JOIN follows ON votes.userid = follows.userid + LEFT JOIN subscriptions ON votes.userid = subscriptions.userid + CROSS JOIN votepoints + WHERE bans.userid IS NULL + GROUP BY url; + +ALTER TABLE users DROP COLUMN isfollower, DROP COLUMN issubscriber; + +INSERT INTO cron (jobName,runInterval) VALUES +('processFollows','30 minutes'), +('processSubscriptions','30 minutes'); + +COMMIT; diff --git a/db/upgrade/v0.5-v0.6.sql b/db/upgrade/v0.5-v0.6.sql new file mode 100644 index 0000000..a43869c --- /dev/null +++ b/db/upgrade/v0.5-v0.6.sql @@ -0,0 +1,5 @@ +BEGIN; + +UPDATE version SET minor = 6; + +COMMIT; diff --git a/db/upgrade/v0.6-v0.7.sql b/db/upgrade/v0.6-v0.7.sql new file mode 100644 index 0000000..0939227 --- /dev/null +++ b/db/upgrade/v0.6-v0.7.sql @@ -0,0 +1,24 @@ +BEGIN; + +UPDATE version SET minor = 7; + +ALTER TABLE votes + ALTER COLUMN requestUrl SET NOT NULL, + ALTER COLUMN userId SET NOT NULL; + +CREATE OR REPLACE PROCEDURE clear_zero_votes() + LANGUAGE SQL + AS $$ + DELETE FROM requests WHERE NOT EXISTS + (SELECT FROM votes WHERE requests.url = votes.requesturl); + $$; + +CREATE OR REPLACE PROCEDURE delete_vote(url varchar,voteuser int) + LANGUAGE SQL + AS $$ + DELETE FROM votes WHERE requesturl = url AND userid = voteuser; + CALL update_scores(); + CALL clear_zero_votes(); + $$; + +COMMIT; diff --git a/db/upgrade/v0.7-v0.8.sql b/db/upgrade/v0.7-v0.8.sql new file mode 100644 index 0000000..af83dae --- /dev/null +++ b/db/upgrade/v0.7-v0.8.sql @@ -0,0 +1,35 @@ +BEGIN; + +UPDATE version SET minor = 8; + +ALTER TABLE config + ALTER COLUMN normaluservotepoints SET DEFAULT 10, + ALTER COLUMN followervotepoints SET DEFAULT 50, + ALTER COLUMN subscribervotepoints SET DEFAULT 100, + ALTER COLUMN title SET DEFAULT '{username}''s Learn Request Queue', + ALTER COLUMN colors SET DEFAULT '{"bg": {"primary": "#444444","table": "#282828","navbar": "#666666","error": "#ff0000"},"fg": {"primary": "#dddddd","ahover": "#ffffff","title": "#eeeeee"}}', + ADD COLUMN normaluserratelimit int NOT NULL DEFAULT 1, + ADD COLUMN followerratelimit int NOT NULL DEFAULT 2, + ADD COLUMN subscriberratelimit int NOT NULL DEFAULT 3; + +CREATE OR REPLACE VIEW ratelimit_vw AS + SELECT users.userid,COALESCE(count,0),ratelimit.reqcount AS max,COALESCE(count,0) >= ratelimit.reqcount AS status + FROM users + LEFT JOIN (SELECT requester,COUNT(url) + FROM requests + WHERE reqtimestamp > (now() - '24 hours'::interval) + GROUP BY requests.requester + ) AS requests ON users.userid = requests.requester + LEFT JOIN follows ON requests.requester = follows.userid + LEFT JOIN subscriptions ON requests.requester = subscriptions.userid + CROSS JOIN config + CROSS JOIN LATERAL (VALUES ( + CASE + WHEN follows.userid IS NULL AND subscriptions.userid IS NULL THEN config.normaluserratelimit + WHEN follows.userid IS NOT NULL AND subscriptions.userid IS NULL THEN config.followerratelimit + WHEN subscriptions.userid IS NOT NULL THEN config.subscriberratelimit + END + )) AS ratelimit(reqcount); + + +COMMIT; diff --git a/docker-compose.yml b/docker-compose.yml index aa08979..818df75 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3.8' services: app: - container_name: learn-request-queue + container_name: lrq build: . depends_on: - db @@ -11,7 +11,7 @@ services: - "80:3000" env_file: .env db: - container_name: learn-request-queue-db + container_name: lrqdb image: postgres ports: - "5432:5432" diff --git a/public/main.js b/public/main.js index d4cf22c..e9fee5d 100644 --- a/public/main.js +++ b/public/main.js @@ -1,28 +1,52 @@ var requestsDiv = document.getElementById("requests"); +var cronJobs = ['processBans']; +var currentPage = 1; +var totalPages = 1; +var count = document.getElementById("count").value; +var sortBy = document.getElementById("sortBy").value; +var sortDir = "desc"; -function getRequests(count,allRequests) { - var reqUrl; - if (allRequests) { - reqUrl = "/api/getAllRequests"; - } else { - reqUrl = "/api/getRequests"; - } - reqUrl += `?count=${count}`; +function getRequests(offset,allRequests) { + if (allRequests) var reqUrl = "/api/getAllRequests"; + else var reqUrl = "/api/getRequests"; + reqUrl += `?count=${count}&offset=${offset}&sort=${sortBy}&sortDirection=${sortDir}`; fetch(reqUrl) .then(response => response.json()) .then(requests => { - window.requests = requests; - buildTable(); + buildTable(requests); }); } -function buildTable() { - var requestsDivHTML = ''; +function buildTable(requests) { + totalPages = Math.ceil(requests.total/document.getElementById("count").value); + document.getElementById("totalPages").innerText = totalPages; + if (currentPage <= 1) { + currentPage = 1; + document.getElementById("pageBtnFirst").disabled = true; + document.getElementById("pageBtnPrev").disabled = true; + } else { + document.getElementById("pageBtnFirst").disabled = false; + document.getElementById("pageBtnPrev").disabled = false; + } + if (currentPage >= totalPages) { + currentPage = totalPages; + document.getElementById("pageBtnLast").disabled = true; + document.getElementById("pageBtnNext").disabled = true; + } else { + document.getElementById("pageBtnLast").disabled = false; + document.getElementById("pageBtnNext").disabled = false; + } + document.getElementById("page").innerHTML = ""; + for (i = 1; i <= totalPages; i++) { + document.getElementById("page").innerHTML += ``; + } + document.getElementById("page").value = currentPage; + var requestsDivHTML = '
SongRequesterScore
'; requestsDivHTML += '' requestsDivHTML += ""; - for (request of requests) { + for (request of requests.requests) { requestsDivHTML += `\ \ `; @@ -44,12 +68,14 @@ function buildTable() { } function updateTable() { - allRequests = document.getElementById("allRequests").checked; - getRequests(document.getElementById("count").value,allRequests); + var offset = (currentPage - 1) * count; + var allRequests = document.getElementById("allRequests").checked; + getRequests(offset,allRequests); } function applyUrlTransforms(url) { console.log("Begin applyUrlTransforms:" + url); + url = url.trim(); if (url.match(/^https?:\/\/(www\.)?youtu(\.be|be\.com)\//)) { // Youtube console.log("Youtube"); var videoid = ""; @@ -65,6 +91,27 @@ function applyUrlTransforms(url) { } } +function goToPage(page) { + currentPage = parseInt(page,10); + updateTable(); +} + +function getColorObject() { + return { + bg: { + primary: document.getElementById('color-bg-primary').value, + table: document.getElementById('color-bg-table').value, + navbar: document.getElementById('color-bg-navbar').value, + error: document.getElementById('color-bg-error').value, + }, + fg: { + primary: document.getElementById('color-fg-primary').value, + ahover: document.getElementById('color-fg-ahover').value, + title: document.getElementById('color-fg-title').value, + } + } +} + function addRequestErr(msg) { document.getElementById('addRequestError').style.display = "inline-block"; document.getElementById('addRequestError').innerText = msg; @@ -85,33 +132,46 @@ function updateRequestErrReset() { document.getElementById('updateRequestError').innerText = ""; } -function showMessage(msg) { - document.getElementById("messageModalText").innerText = msg; +function streamerSettingsErr(msg) { + document.getElementById('streamerSettingsError').style.display = "inline-block"; + document.getElementById('streamerSettingsError').innerText = msg; +} + +function streamerSettingsErrReset() { + document.getElementById('streamerSettingsError').style.display = "none"; + document.getElementById('streamerSettingsError').innerText = ""; +} + +// Hides all modals in preparation to show a one or to close out of +// all modals and return to the page. Does NOT hide modalBackground. +function hideModals() { + document.getElementById("messageModal").style.display = "none"; document.getElementById("addRequestModal").style.display = "none"; document.getElementById("updateRequestModal").style.display = "none"; document.getElementById("deleteRequestModal").style.display = "none"; + document.getElementById("streamerSettingsModal").style.display = "none"; +} + +function closeAllModals() { + hideModals(); + document.getElementById("modalBackground").style.display = "none"; +} + +function showMessage(msg) { + hideModals(); + document.getElementById("messageModalText").innerText = msg; document.getElementById("modalBackground").style.display = "flex"; document.getElementById("messageModal").style.display = "block"; } -function closeMessageModal() { - document.getElementById("modalBackground").style.display = "none"; - document.getElementById("messageModal").style.display = "none"; -} - function openAddRequestModal() { + hideModals(); document.getElementById("modalBackground").style.display = "flex"; - document.getElementById("updateRequestModal").style.display = "none"; - document.getElementById("messageModal").style.display = "none"; document.getElementById("addRequestModal").style.display = "block"; } -function closeAddRequestModal() { - document.getElementById("modalBackground").style.display = "none"; - document.getElementById("addRequestModal").style.display = "none"; -} - function openUpdateRequestModal(tr) { + hideModals(); var url = tr.getElementsByClassName('request-link')[0].firstChild.href; var score = tr.getElementsByClassName('request-score')[0].innerText; var state = tr.getElementsByClassName('request-state')[0].innerText; @@ -120,18 +180,12 @@ function openUpdateRequestModal(tr) { document.getElementById("updateRequestModalCurrentScore").innerText = score; document.querySelector(`#updateRequestStateSelect [value="${state}"]`).selected = true; document.getElementById("scoreModifierInput").value = 0; - document.getElementById("messageModal").style.display = "none"; - document.getElementById("addRequestModal").style.display = "none"; document.getElementById("modalBackground").style.display = "flex"; document.getElementById("updateRequestModal").style.display = "block"; } -function closeUpdateRequestModal() { - document.getElementById("modalBackground").style.display = "none"; - document.getElementById("updateRequestModal").style.display = "none"; -} - function openDeleteRequestModal(url) { + hideModals(); document.getElementById("updateRequestUrl").href = url; document.getElementById("updateRequestUrl").innerText = url; document.getElementById("messageModal").style.display = "none"; @@ -141,17 +195,21 @@ function openDeleteRequestModal(url) { document.getElementById("deleteRequestModal").style.display = "block"; } +// Returns to update request modal function closeDeleteRequestModal() { + hideModals(); document.getElementById("deleteRequestModal").style.display = "none"; document.getElementById("updateRequestModal").style.display = "block"; } -function closeAllModals() { - document.getElementById("messageModal").style.display = "none"; - document.getElementById("addRequestModal").style.display = "none"; - document.getElementById("updateRequestModal").style.display = "none"; - document.getElementById("deleteRequestModal").style.display = "none"; - document.getElementById("modalBackground").style.display = "none"; +function openStreamerSettingsModal() { + hideModals(); + document.getElementById("modalBackground").style.display = "flex"; + document.getElementById("streamerSettingsModal").style.display = "block"; +} + +function cronRequest(job) { + if (!cronJobs.includes(job)) throw new Error("Request for invalid job"); } const validUrlRegexes = [ @@ -178,7 +236,7 @@ function validateAndSubmitRequest() { updateTable(); document.getElementById("addRequestUrl").value = ""; response.text().then((message) => { - closeAddRequestModal(); + closeAllModals(); showMessage(message); }); }); @@ -262,14 +320,55 @@ function deleteRequest(url) { }); } +function updatePageTitle(pageTitle) { + streamerSettingsErrReset(); + fetch("/api/updatePageTitle", { method: 'POST', body: new URLSearchParams({ + pageTitle: pageTitle + })}) + .then(response => { + if (response.ok) { + location.reload(); + } else { + response.text().then(streamerSettingsErr); + } + }); +} +function updateColors(colors) { + streamerSettingsErrReset(); + fetch("/api/updateColors", { method: 'POST', headers: { + 'Content-Type': 'application/json' + }, body: JSON.stringify(colors)}) + .then(response => { + if (response.ok) { + location.reload(); + } else { + response.text().then(streamerSettingsErr); + } + }); +} + +function updateVotePoints(user,follower,subscriber) { + streamerSettingsErrReset(); + fetch("/api/updateVotePoints", { method: 'POST', body: new URLSearchParams({ + user: user, + follower: follower, + subscriber: subscriber + })}) + .then(response => { + if (response.ok) { + location.reload(); + } else { + response.text().then(streamerSettingsErr); + } + }); +} updateTable(); document.addEventListener("keydown", function onEvent(event) { if (event.key === "Escape") { - closeMessageModal(); - closeAddRequestModal(); + closeAllModals(); } }); document.getElementById("modalBackground").addEventListener("click", (e) => { if (e.target === e.currentTarget) closeAllModals();}); @@ -279,5 +378,16 @@ for(state of validStates) { var opt = document.createElement("option"); opt.text = state; opt.value = state; - updateRequestStateSelect.add(opt) + updateRequestStateSelect.add(opt); +} + +function toggleSortDir() { + if (window.sortDir == "desc") { + document.getElementById("sortDir").innerText = "↑"; + window.sortDir = "asc"; + } else { + document.getElementById("sortDir").innerText = "↓"; + window.sortDir = "desc"; + } + updateTable(); } diff --git a/public/style.css b/public/style.css index 7f4cbba..ccea461 100644 --- a/public/style.css +++ b/public/style.css @@ -12,6 +12,12 @@ button, input, select { font-size: 100%; } +input[type="color"] { + border: none; + padding: 0px; + vertical-align: middle; +} + a { color: #ddd; } @@ -105,6 +111,20 @@ div#nav-userpic { text-align: right; } +.tableSettings { + display: flex; + align-items: center; + justify-content: space-between; +} + +.tableSettings > span { + margin: auto; +} + +#tableSettingsTop { + margin-bottom: 10px; +} + #modalBackground { display: none; position: fixed; @@ -117,6 +137,7 @@ div#nav-userpic { height: 100%; background-color: #444; background-color: #444a; + margin: 5px; } .modal { @@ -179,3 +200,18 @@ div#nav-userpic { #deleteRequestLink { color: #f00; } + +#streamerSettingsMain { + max-height: 65vh; + overflow-y: scroll; +} + +#votepoints { + padding: 0 1.5em; + display: flex; + justify-content: space-between; +} + +#sortDir { + padding: 0; +} diff --git a/src/app.ts b/src/app.ts index e67ab26..583a56e 100644 --- a/src/app.ts +++ b/src/app.ts @@ -2,13 +2,14 @@ import * as config from "./config"; import * as requests from "./requests"; import * as twitch from "./twitch"; import * as queries from "./queries"; -import { log, LogLevel } from "./logging" import { URLSearchParams } from "url"; import express from "express"; import session from "express-session"; import pg from "pg"; import pgSessionStore from "connect-pg-simple"; import fetch, { Response as FetchResponse } from "node-fetch"; +import { log, LogLevel } from "./logging" +import cron from "./cron"; import db from "./db"; import errorHandler from "./errors"; import * as version from "./version"; @@ -26,8 +27,10 @@ async function validateApiToken(session: Express.Session) { } const app = express(); +app.use(version.checkVersionMiddleware); app.use(express.static('public')); app.use(express.urlencoded({extended: false})); +app.use(express.json()); app.use(session({ secret: config.sessionSecret, saveUninitialized: false, @@ -39,31 +42,85 @@ app.use(session({ // API app.get("/api/getRequests", async (request, response) => { - if (!request.session) { - throw new Error ("Missing request.session") - } - var requestCount = ( request.query.count ? parseInt(request.query.count as string, 10) : 5 ); + if (!request.session) throw new Error ("Missing request.session"); await validateApiToken(request.session); + var requestCount = ( request.query.count ? parseInt(request.query.count as string, 10) : 5 ); + var requestOffset = ( request.query.offset ? parseInt(request.query.offset as string, 10) : 0 ); + var sortDirection = ( request.query.sortDirection == "asc" ? "ASC" : "DESC" ); + var inverseSortDirection = ( sortDirection == "ASC" ? "DESC" : "ASC" ); + switch (request.query.sort) { + case undefined: // Default sort by newest + case "timestamp": + var requestSort = `reqTimestamp ${sortDirection}`; + break; + case "score": + var requestSort = `score ${sortDirection}, reqTimestamp ${inverseSortDirection}`; + break; + case "title": + var requestSort = `title ${sortDirection}` + break; + case "requester": + var requestSort = `requester ${sortDirection}, title ${sortDirection}` + break; + default: + response.status(400); + response.send("Invalid sort"); + return; + }; + var requestsTotal = await requests.getRequestsTotal(); if (request.session.user) { - requests.getRequestsVoted(requestCount,request.session.user.id).then((val: Array) => response.send(val)) + requests.getRequestsVoted(requestCount,requestOffset,requestSort,request.session.user.id) + .then((val: Array) => response.send({ + total: requestsTotal, + requests: val + })) .catch((e: any) => errorHandler(request,response,e)); } else { - requests.getRequests(requestCount).then((val: Array) => response.send(val)) + requests.getRequests(requestCount,requestOffset,requestSort) + .then((val: Array) => response.send({ + total: requestsTotal, + requests: val + })) .catch((e: any) => errorHandler(request,response,e)); } }); app.get("/api/getAllRequests", async (request, response) => { - if (!request.session) { - throw new Error ("Missing request.session") - } - var requestCount = ( request.query.count ? parseInt(request.query.count as string, 10) : 5 ); + if (!request.session) throw new Error ("Missing request.session"); await validateApiToken(request.session); + var requestCount = ( request.query.count ? parseInt(request.query.count as string, 10) : 5 ); + var requestOffset = ( request.query.offset ? parseInt(request.query.offset as string, 10) : 0 ); + var sortDirection = ( request.query.sortDirection == "asc" ? "ASC" : "DESC" ); + switch (request.query.sort) { + case undefined: + case "score": + var requestSort = `score ${sortDirection}, reqTimestamp ASC`; + break; + case "timestamp": + var requestSort = `reqTimestamp ${sortDirection}`; + break; + case "alpha": + var requestSort = `title ${sortDirection}` + break; + default: + response.status(400); + response.send("Invalid sort"); + return; + } + var requestsTotal = await requests.getAllRequestsTotal(); if (request.session.user) { - requests.getAllRequestsVoted(requestCount,request.session.user.id).then((val: Array) => response.send(val)) + requests.getAllRequestsVoted(requestCount,requestOffset,requestSort,request.session.user.id) + .then((val: Array) => response.send({ + total: requestsTotal, + requests: val + })) .catch((e: any) => errorHandler(request,response,e)); } else { - requests.getAllRequests(requestCount).then((val: Array) => response.send(val)) + requests.getAllRequests(requestCount,requestOffset,requestSort) + .then((val: Array) => response.send({ + total: requestsTotal, + requests: val + })) .catch((e: any) => errorHandler(request,response,e)); } }); @@ -76,7 +133,8 @@ app.post("/api/addRequest", async (request, response) => { response.send("Session expired; please log in again"); return; } - var banned = await db.query(Object.assign(queries.checkBan, { values: [request.session.user.id] })).then((result: pg.QueryResult) => result.rowCount > 0); + var banned = await db.query(Object.assign(queries.checkBan, { values: [request.session.user.id] })) + .then((result: pg.QueryResult) => result.rowCount > 0); if (banned) { response.status(401); response.send("You are banned; you may not add new requests."); @@ -187,6 +245,139 @@ app.post("/api/updateRequestScoreModifier", async (request, response) => { .catch((e: any) => errorHandler(request,response,e)); }); +app.post("/api/updatePageTitle", async (request, response) => { + if (request.session) await validateApiToken(request.session); + if (!request.session || !request.session.user) { + response.status(401); + response.send("Session expired; please log in again"); + return; + } + var streamerid = await db.query(queries.getStreamerId).then((result: pg.QueryResult) => result.rows[0]['userid']); + if (request.session.user.id != streamerid) { + response.status(401); + response.send("You are not the streamer"); + return; + } + if (!request.body.pageTitle) { + response.status(400); + response.send("Missing pageTitle"); + return; + } + var pageTitle = request.body.pageTitle as string; + response.type('text/plain'); + pageTitle = pageTitle.replace(/[&<>"']/g, function(m) { + switch (m) { + case '&': + return '&'; + case '<': + return '<'; + case '>': + return '>'; + case '"': + return '"'; + case "'": + return '''; + default: + return ''; + } + }); + await db.query(Object.assign(queries.updatePageTitle,{ values: [pageTitle] })) + .catch((e: any) => errorHandler(request,response,e)); + response.status(200); + response.send('Successfully updated page title'); +}); + +app.post("/api/updateColors", async (request, response) => { + if (request.session) await validateApiToken(request.session); + if (!request.session || !request.session.user) { + response.status(401); + response.send("Session expired; please log in again"); + return; + } + var streamerid = await db.query(queries.getStreamerId).then((result: pg.QueryResult) => result.rows[0]['userid']); + if (request.session.user.id != streamerid) { + response.status(401); + response.send("You are not the streamer"); + return; + } + if (!request.body.bg) { + response.status(400); + response.send("Missing bg"); + return; + } + if (!request.body.fg) { + response.status(400); + response.send("Missing fg"); + return; + } + type Colors = { [key: string]: { [key: string]: string} } + var colors: Colors = { bg: {}, fg: {} }; + console.log(JSON.stringify(request.body,null,2)); + for (var color of ['primary','table','navbar','error']) { + var setcolor = request.body.bg[color]; + if (/^#[0-9a-fA-F]{6}$/.test(setcolor)) { + colors.bg[color] = setcolor + } else { + response.status(400); + response.send(`Color 'bg.${color}' missing or invalid`) + return; + } + } + for (var color of ['primary','ahover','title']) { + var setcolor = request.body.fg[color]; + if (/^#[0-9a-fA-F]{6}$/.test(setcolor)) { + colors.fg[color] = setcolor + } else { + response.status(400); + response.send(`Color 'fg.${color}' missing or invalid`) + return; + } + } + response.type('text/plain'); + await db.query(Object.assign(queries.updateColors,{ values: [JSON.stringify(colors)] })) + .catch((e: any) => errorHandler(request,response,e)); + response.status(200); + response.send('Successfully updated colors'); +}); + +app.post("/api/updateVotePoints", async (request, response) => { + if (request.session) await validateApiToken(request.session); + if (!request.session || !request.session.user) { + response.status(401); + response.send("Session expired; please log in again"); + return; + } + var streamerid = await db.query(queries.getStreamerId).then((result: pg.QueryResult) => result.rows[0]['userid']); + if (request.session.user.id != streamerid) { + response.status(401); + response.send("You are not the streamer"); + return; + } + if (!request.body.user) { + response.status(400); + response.send("Missing user"); + return; + } + if (!request.body.follower) { + response.status(400); + response.send("Missing follower"); + return; + } + if (!request.body.subscriber) { + response.status(400); + response.send("Missing subscriber"); + return; + } + var user = request.body.user as number; + var follower = request.body.follower as number; + var subscriber = request.body.subscriber as number; + response.type('text/plain'); + await db.query(Object.assign(queries.updateVotePoints,{ values: [user,follower,subscriber] })) + .catch((e: any) => errorHandler(request,response,e)); + response.status(200); + response.send('Successfully updated page title'); +}); + app.post("/api/deleteRequest", async (request, response) => { if (request.session) await validateApiToken(request.session); if (!request.session || !request.session.user) { @@ -264,6 +455,37 @@ app.post("/api/deleteVote", async (request,response) => { .catch((e: any) => errorHandler(request,response,e)); }); +app.get("/api/cronRequest", async (request, response) => { + if (request.session) await validateApiToken(request.session); + if (!request.session || !request.session.user) { + response.status(401); + response.send("Session expired; please log in again"); + return; + } + var streamerid = await db.query(queries.getStreamerId).then((result: pg.QueryResult) => result.rows[0]['userid']); + if (request.session.user.id != streamerid) { + response.status(401); + response.send("You are not the streamer"); + return; + } + if (!request.query.job) { + response.status(400); + response.send("Missing job"); + return; + } + var job = request.body.job as string; + try { + cron.validateJob(job) + } catch (e) { + response.status(400); + response.send("Invalid job") + return; + } + response.type('text/plain'); + cron.request(job).catch((e: any) => errorHandler(request,response,e)); +}); + + // Twitch callback app.get("/callback", async (request, response) => { if (request.query.error) { @@ -300,7 +522,7 @@ app.get("/callback", async (request, response) => { app.get("/", async (request, response) => { if (request.session) await validateApiToken(request.session); var streamerInfo = await db.query(queries.getStreamerInfo).then((result: pg.QueryResult) => result.rows[0]); - var validStates = JSON.stringify((await db.query(queries.getValidStates).then((result: pg.QueryResult) => result.rows)).map((row: any) => row.state)); + var streamerConfig = await db.query(queries.getConfig).then((result: pg.QueryResult) => result.rows[0]); if (typeof streamerInfo == 'undefined') { response.redirect(307, `https://id.twitch.tv/oauth2/authorize?client_id=${config.twitchClientId}&redirect_uri=${config.urlPrefix}/callback&response_type=code&scope=channel:read:subscriptions moderation:read`); return; @@ -311,9 +533,11 @@ app.get("/", async (request, response) => { clientId: config.twitchClientId, urlPrefix: config.urlPrefix, streamerName: streamerInfo['displayname'], - streamerProfilePicture: streamerInfo['imageurl'] + streamerProfilePicture: streamerInfo['imageurl'], + config: streamerConfig, }); } else { + var validStates = JSON.stringify((await db.query(queries.getValidStates).then((result: pg.QueryResult) => result.rows)).map((row: any) => row.state)); response.render('main.eta', { loggedIn: true, userName: request.session.user.display_name, @@ -321,72 +545,36 @@ app.get("/", async (request, response) => { validStates: validStates, isStreamer: streamerInfo['userid'] == request.session.user.id, streamerName: streamerInfo['displayname'], - streamerProfilePicture: streamerInfo['imageurl'] + streamerProfilePicture: streamerInfo['imageurl'], + config: streamerConfig, }); } }); +app.get("/colors.css", async (_request, response) => { + var streamerInfo = await db.query(queries.getStreamerInfo).then((result: pg.QueryResult) => result.rows[0]); + var colors = await db.query(queries.getConfig).then((result: pg.QueryResult) => result.rows[0]['colors']); + if (typeof streamerInfo == 'undefined') return; + response.contentType("text/css"); + response.render('colors.eta', colors); +}); + // Streamer Panel //app.get("/streamer/", async (request, response) => { // //}); // Logout -app.get("/logout", async (request, response) => request.session!.destroy(() => response.redirect(307, '/'))); - -async function processBannedUsers() { - log(LogLevel.INFO,"processBannedUsers run at " + new Date().toISOString()); - var streamer = await db.query(queries.getStreamerIdToken).then((result: pg.QueryResult) => result.rows[0]); - if (typeof streamer == 'undefined') return; - var response = await twitch.streamerApiRequest("/moderation/banned?broadcaster_id=" + streamer['userid']); - var dbconn = await db.connect(); - try { - await dbconn.query('BEGIN'); - dbconn.query("DELETE FROM bans"); - log(LogLevel.DEBUG,"Ban list:") - log(LogLevel.DEBUG,JSON.stringify(response.data,null,2)); - var insertBanQuery = "INSERT INTO bans (userid) VALUES "; - var banRow = 0; - var bansArray: number[] = []; - for (var ban of response.data) { - if (ban.expires_at == '') { - banRow++; - insertBanQuery += `($${banRow}), `; - bansArray.push(ban.user_id as number); - } - } - var banQueryConfig = { - text: insertBanQuery.slice(0,-2), // Cut last `, ` off of the end - values: bansArray - }; - log(LogLevel.DEBUG,"banQueryConfig object:") - log(LogLevel.DEBUG,JSON.stringify(banQueryConfig,null,2)) - dbconn.query(banQueryConfig); - dbconn.query("CALL update_scores()"); - await dbconn.query('COMMIT'); - } catch (e) { - await dbconn.query('ROLLBACK'); - } finally { - dbconn.release(); - } - setTimeout(processBannedUsers,3600000+Math.floor(Math.random()*900000)) // Run every 1-1.25 hours to balance load -} - -setTimeout(processBannedUsers,600000+Math.floor(Math.random()*600000)) - -async function processEmptyMetadata() { - log(LogLevel.INFO,"processEmptyMetadata run at " + new Date().toISOString()); - var result = await db.query(queries.getRequestsWithEmptyMetadata); - for (var row of result.rows) { - log(LogLevel.DEBUG,"Processing empty metadata for request: " + row['url']); - requests.updateRequestMetadata(row['url']); - } - setTimeout(processEmptyMetadata,3600000+Math.floor(Math.random()*900000)) // Run every 1-1.25 hours to balance load -} - -processEmptyMetadata(); +app.get("/logout", async (request, response) => + request.session!.destroy(() => response.redirect(307, '/'))); // Check version then listen version.checkVersion().then(_ => app.listen(config.port, () => { + cron.run(); + setInterval(cron.run,config.cronInterval); console.log(`Listening on port ${config.port}`); -})); +})) +.catch((e) => { + log(LogLevel.ERROR,e) + process.exit(1); +}); diff --git a/src/config.ts b/src/config.ts index ce3e768..a7ab483 100644 --- a/src/config.ts +++ b/src/config.ts @@ -36,6 +36,13 @@ if (typeof process.env.YOUTUBE_SECRET == 'undefined') { } export const youtubeSecret: string = process.env.YOUTUBE_SECRET!; -export const logLevel = (process.env.LOG_LEVEL ? LogLevel[process.env.LOG_LEVEL as keyof typeof LogLevel] : LogLevel.ERROR ); +export const logLevel = (process.env.LOG_LEVEL + ? LogLevel[process.env.LOG_LEVEL as keyof typeof LogLevel] + : LogLevel.ERROR ); + +export const cronInterval: number = (process.env.CRON_INTERVAL + ? parseInt(process.env.CRON_INTERVAL as string, 10) + : 60000); + console.log("Running with logLevel = " + logLevel) diff --git a/src/cron.ts b/src/cron.ts new file mode 100644 index 0000000..fef3758 --- /dev/null +++ b/src/cron.ts @@ -0,0 +1,70 @@ +import cronjobs from "./cronjobs" +import * as queries from "./queries"; +import * as twitch from "./twitch"; +import { log, LogLevel } from "./logging" +import db from "./db"; +import pg from "pg"; + +interface CronJob { + (streamer: twitch.StreamerUserIdTokenPair): Promise +} + +function validateJob(job: string) { + if (!Object.keys(cronjobs).includes(job)) throw new Error("Invalid cronjob " + job); +} + +async function runJob(job: string) { + validateJob(job); + var streamer = await db.query(queries.getStreamerIdToken).then( + (result: pg.QueryResult) => result.rows[0]); + return await (cronjobs as { [key: string]: CronJob })[job](streamer); +} + +// Run a specific job on request of streamer +async function request(job: string) { + validateJob(job); + // TODO: Rate limiting + await runJob(job); +} + +async function run() { + // If instance is not yet set up, end processing at this point + var streamer = await db.query(queries.getStreamerIdToken).then( + (result: pg.QueryResult) => result.rows[0]); + if (typeof streamer == 'undefined') { + log(LogLevel.INFO,"Cron run skipped due to instance not being set up") + return; + } + log(LogLevel.INFO,"Begin cron run") + for (var job in cronjobs) { + log(LogLevel.INFO,"cron: Checking job " + job); + var dbconn = await db.connect(); + await dbconn.query("BEGIN"); + try { + var needsRun = (await dbconn.query( + Object.assign(queries.getAndLockCronJob,{values: [job]}) + )).rowCount; + if (needsRun) { + log(LogLevel.INFO,`cron: Database says job ${job} needs run; executing`); + await (cronjobs as { [key: string]: CronJob })[job](streamer) + log(LogLevel.INFO,`cron: Job ${job} returned without exception; updating lastSuccess`); + await dbconn.query(Object.assign(queries.updateCronJobLastSuccess,{values: [job]})) + await dbconn.query("COMMIT"); + } else { + log(LogLevel.INFO,`cron: Database says job ${job} does not need run; skipping`); + await dbconn.query("ROLLBACK"); + } + } catch(e) { + log(LogLevel.ERROR,`cron: Job ${job} threw exception; rolling back`); + await dbconn.query("ROLLBACK"); + log(LogLevel.ERROR,`cron: Job ${job} exception message: ${e}`) + } finally { + log(LogLevel.DEBUG,`cron: Job ${job} hit finally; releasing dbconn`); + dbconn.release(); + } + log(LogLevel.INFO,"cron: Finished job " + job); + } + log(LogLevel.INFO,"End cron run") +} + +export = {run,validateJob,request} diff --git a/src/cronjobs/index.ts b/src/cronjobs/index.ts new file mode 100644 index 0000000..94dd385 --- /dev/null +++ b/src/cronjobs/index.ts @@ -0,0 +1,11 @@ +import { processBans } from './processBans'; +import { processFollows } from './processFollows'; +import { processSubscriptions } from './processSubscriptions'; +import { processEmptyMetadata } from './processEmptyMetadata'; + +export = { + processBans, + processFollows, + processSubscriptions, + processEmptyMetadata +} diff --git a/src/cronjobs/processBans.ts b/src/cronjobs/processBans.ts new file mode 100644 index 0000000..a47a2fa --- /dev/null +++ b/src/cronjobs/processBans.ts @@ -0,0 +1,58 @@ +import * as twitch from "../twitch"; +import { log, LogLevel } from "../logging" +import db from "../db"; + +export async function processBans(streamer: twitch.StreamerUserIdTokenPair) { + var dbconn = await db.connect(); + try { + await dbconn.query('BEGIN'); + await dbconn.query("DELETE FROM bans"); + var response = await twitch.streamerApiRequest(streamer, + `/moderation/banned?broadcaster_id=${streamer.userid}&first=100`); + log(LogLevel.DEBUG,"Ban API response:") + log(LogLevel.DEBUG,JSON.stringify(response,null,2)); + while (true) { + var insertBanQuery = "INSERT INTO bans (userid) VALUES "; + var banRow = 0; // Used for $1, $2, etc. in parameterized query + var bansArray: number[] = []; + if (Object.keys(response.data).length > 0) { + for (var ban of response.data) { + if (ban.expires_at == '') { + banRow++; + insertBanQuery += `($${banRow}), `; + bansArray.push(ban.user_id as number); + } + } + insertBanQuery = insertBanQuery.slice(0,-2); // Cut last `, ` off of the end + insertBanQuery += " ON CONFLICT DO NOTHING"; // Deal with broken endpoint returning dupes + var banQueryConfig = { + text: insertBanQuery, + values: bansArray + }; + log(LogLevel.DEBUG,"banQueryConfig object:") + log(LogLevel.DEBUG,JSON.stringify(banQueryConfig,null,2)) + await dbconn.query(banQueryConfig); + } + if (response.pagination.cursor) { + var oldFirstUserid = response.data[0].user_id; + response = await twitch.streamerApiRequest(streamer, + `/moderation/banned?broadcaster_id=${streamer.userid}&after=${response.pagination.cursor}&first=100`); + log(LogLevel.DEBUG,"Ban API response:"); + log(LogLevel.DEBUG,JSON.stringify(response,null,2)); + // Work around broken api endpoint giving a cursor referring to the + // current page, causing an infinite loop + if (oldFirstUserid == response.data[0].user_id) break; + } else { + break; + } + } + await dbconn.query("CALL update_scores()"); + await dbconn.query('COMMIT'); + } catch (e) { + log(LogLevel.ERROR,"cronjobs.processBans: Exception thrown; rolling back"); + await dbconn.query('ROLLBACK'); + throw(e); + } finally { + await dbconn.release(); + } +} diff --git a/src/cronjobs/processEmptyMetadata.ts b/src/cronjobs/processEmptyMetadata.ts new file mode 100644 index 0000000..9b911bb --- /dev/null +++ b/src/cronjobs/processEmptyMetadata.ts @@ -0,0 +1,12 @@ +import * as requests from "../requests"; +import * as queries from "../queries"; +import { log, LogLevel } from "../logging" +import db from "../db"; + +export async function processEmptyMetadata(_: any) { // TODO: Consume streamer userid for WHERE clause + var result = await db.query(queries.getRequestsWithEmptyMetadata); + for (var row of result.rows) { + log(LogLevel.DEBUG,"Processing empty metadata for request: " + row['url']); + requests.updateRequestMetadata(row['url']); + } +} diff --git a/src/cronjobs/processFollows.ts b/src/cronjobs/processFollows.ts new file mode 100644 index 0000000..9d74330 --- /dev/null +++ b/src/cronjobs/processFollows.ts @@ -0,0 +1,53 @@ +import * as twitch from "../twitch"; +import { log, LogLevel } from "../logging" +import db from "../db"; + +export async function processFollows(streamer: twitch.StreamerUserIdTokenPair) { + var dbconn = await db.connect(); + try { + await dbconn.query('BEGIN'); + await dbconn.query("DELETE FROM follows"); + // Insert the streamer as a follower + await dbconn.query({text: "INSERT INTO follows (userid) VALUES ($1)", values: [streamer.userid]}) + var response = await twitch.streamerApiRequest(streamer, + `/users/follows?to_id=${streamer.userid}&first=100`); + log(LogLevel.DEBUG,"Follows API response:") + log(LogLevel.DEBUG,JSON.stringify(response,null,2)); + while (true) { + var insertFollowQuery = "INSERT INTO follows (userid) VALUES "; + var followRow = 0; // Used for $1, $2, etc. in parameterized query + var followsArray: number[] = []; + if (Object.keys(response.data).length > 0) { + for (var follow of response.data) { + followRow++; + insertFollowQuery += `($${followRow}), `; + followsArray.push(follow.from_id as number); + } + insertFollowQuery = insertFollowQuery.slice(0,-2); // Cut last `, ` off of the end + var followQueryConfig = { + text: insertFollowQuery, + values: followsArray + }; + log(LogLevel.DEBUG,"followQueryConfig object:") + log(LogLevel.DEBUG,JSON.stringify(followQueryConfig,null,2)) + await dbconn.query(followQueryConfig); + } + if (response.pagination.cursor) { + response = await twitch.streamerApiRequest(streamer, + `/users/follows?to_id=${streamer.userid}&after=${response.pagination.cursor}&first=100`); + log(LogLevel.DEBUG,"Follow API response:"); + log(LogLevel.DEBUG,JSON.stringify(response,null,2)); + } else { + break; + } + } + await dbconn.query("CALL update_scores()"); + await dbconn.query('COMMIT'); + } catch (e) { + log(LogLevel.ERROR,"cronjobs.processFollows: Exception thrown; rolling back"); + await dbconn.query('ROLLBACK'); + throw(e); + } finally { + await dbconn.release(); + } +} diff --git a/src/cronjobs/processSubscriptions.ts b/src/cronjobs/processSubscriptions.ts new file mode 100644 index 0000000..5c53a10 --- /dev/null +++ b/src/cronjobs/processSubscriptions.ts @@ -0,0 +1,55 @@ +import * as twitch from "../twitch"; +import { log, LogLevel } from "../logging" +import db from "../db"; + +export async function processSubscriptions(streamer: twitch.StreamerUserIdTokenPair) { + var dbconn = await db.connect(); + try { + await dbconn.query('BEGIN'); + await dbconn.query("DELETE FROM subscriptions"); + var response = await twitch.streamerApiRequest(streamer, + `/subscriptions?broadcaster_id=${streamer.userid}&first=100`); + log(LogLevel.DEBUG,"Subscriptions API response:") + log(LogLevel.DEBUG,JSON.stringify(response,null,2)); + while (true) { + var insertSubscriptionQuery = "INSERT INTO subscriptions (userid) VALUES "; + var subscriptionRow = 0; // Used for $1, $2, etc. in parameterized query + var subscriptionsArray: number[] = []; + if (Object.keys(response.data).length > 0) { + for (var subscription of response.data) { + subscriptionRow++; + insertSubscriptionQuery += `($${subscriptionRow}), `; + subscriptionsArray.push(subscription.user_id as number); + } + insertSubscriptionQuery = insertSubscriptionQuery.slice(0,-2); // Cut last `, ` off of the end + insertSubscriptionQuery += " ON CONFLICT DO NOTHING"; // Deal with broken endpoint returning dupes + var subscriptionQueryConfig = { + text: insertSubscriptionQuery, + values: subscriptionsArray + }; + log(LogLevel.DEBUG,"subscriptionQueryConfig object:") + log(LogLevel.DEBUG,JSON.stringify(subscriptionQueryConfig,null,2)) + await dbconn.query(subscriptionQueryConfig); + } + if (response.pagination.cursor) { + var oldFirstUserid = response.data[0].user_id; + response = await twitch.streamerApiRequest(streamer, + `/subscriptions?broadcaster_id=${streamer.userid}&after=${response.pagination.cursor}&first=100`); + log(LogLevel.DEBUG,"Subscription API response:"); + log(LogLevel.DEBUG,JSON.stringify(response,null,2)); + if (response.data.length === 0 || oldFirstUserid === response.data[0].user_id) break; + } else { + break; + } + } + await dbconn.query("CALL update_scores()"); + await dbconn.query('COMMIT'); + } catch (e) { + log(LogLevel.ERROR,"cronjobs.processSubscriptions: Exception: " + e); + log(LogLevel.ERROR,"cronjobs.processSubscriptions: Exception thrown; rolling back"); + await dbconn.query('ROLLBACK'); + throw(e); + } finally { + await dbconn.release(); + } +} diff --git a/src/logging.ts b/src/logging.ts index 8ed1376..46db51d 100644 --- a/src/logging.ts +++ b/src/logging.ts @@ -10,5 +10,5 @@ export enum LogLevel { export async function log(logLevel: LogLevel, logMessage: any) { if (config.logLevel >= logLevel) - console.log(LogLevel[logLevel].padStart(7) + ' | ' + logMessage) + console.log(new Date().toISOString() + LogLevel[logLevel].padStart(7) + ' | ' + logMessage) } diff --git a/src/queries.ts b/src/queries.ts index 4f1d3a5..8fa54e6 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -26,40 +26,46 @@ export const getStreamerInfo = { text: "SELECT userid,displayname,imageurl FROM streamer_user_vw" } +export const getConfig = { + name: "getConfig", + text: "SELECT * FROM config" +} + export const updateStreamer = { name: "updateStreamer", text: "INSERT INTO streamer (userid,tokenPair) VALUES ($1,$2)\ ON CONFLICT (userid) DO UPDATE SET tokenPair = $2" } -// Request-related queries -export const getRequests = { - name: "getRequests", - text: "SELECT * FROM requests_vw \ - JOIN states ON requests_vw.state = states.state \ - WHERE active LIMIT $1" +export const updatePageTitle = { + name: "updatePageTitle", + text: "UPDATE config SET title = $1" } -export const getRequestsVoted = { - name: "getRequestsVoted", - text: "SELECT * FROM get_requests_voted($2) \ - JOIN states ON get_requests_voted.state = states.state \ - WHERE active LIMIT $1" +export const updateColors = { + name: "updateColors", + text: "UPDATE config SET colors = $1" } -export const getAllRequests = { - name: "getAllRequests", - text: "SELECT * FROM requests_vw LIMIT $1" +export const updateVotePoints = { + name: "updateVotePoints", + text: "CALL update_vote_points($1,$2,$3)" } -export const getAllRequestsVoted = { - name: "getAllRequestsVoted", - text: "SELECT * FROM get_requests_voted($2) LIMIT $1" +export const getRequestsTotal = { + name: "getRequestsTotal", + text: "SELECT COUNT(*) FROM requests_vw \ + JOIN states ON requests_vw.state = states.state WHERE active" +} + +export const getAllRequestsTotal = { + name: "getAllRequestsTotal", + text: "SELECT COUNT(*) FROM requests_vw" } export const checkRequestExists = { name: "checkRequestExists", - text: "SELECT url FROM requests WHERE url = $1" + text: "SELECT url,requester,state FROM requests_vw WHERE url = $1" } export const addRequest = { @@ -111,3 +117,21 @@ export const getDbVersion = { name: "getDbVersion", text: "SELECT get_version()" } + +export const getAndLockCronJob = { + name: "getCronJob", + text: `SELECT * FROM cron + WHERE (lastSuccess + runInterval) < now() + AND jobName = $1 + FOR UPDATE SKIP LOCKED` +} + +export const updateCronJobLastSuccess = { + name: "updateCronJobLastSuccess", + text: "UPDATE cron SET lastSuccess = now() WHERE jobName = $1" +} + +export const getRateLimitStatus = { + name: "getRateLimitStatus", + text: "SELECT * FROM ratelimit_vw WHERE userid = $1" +} diff --git a/src/requests.ts b/src/requests.ts index d23611e..1a35094 100644 --- a/src/requests.ts +++ b/src/requests.ts @@ -4,30 +4,58 @@ import { log, LogLevel } from "./logging" import pg from "pg"; import db from "./db"; -export async function getRequests(count: number) { - var query = Object.assign(queries.getRequests, { values: [count] }); +export async function getRequests(count: number, offset: number, sort: string) { + var query = { + text: "SELECT * FROM requests_vw \ + JOIN states ON requests_vw.state = states.state WHERE active \ + ORDER BY " + sort + " LIMIT $1 OFFSET $2", + values: [count,offset] + }; return db.query(query) .then((result: pg.QueryResult) => result.rows); }; -export async function getAllRequests(count: number) { - var query = Object.assign(queries.getAllRequests, { values: [count] }); +export async function getRequestsVoted(count: number, offset: number, sort: string, user: number) { + var query = { + text: "SELECT * FROM get_requests_voted($3) \ + JOIN states ON get_requests_voted.state = states.state WHERE active \ + ORDER BY " + sort + " LIMIT $1 OFFSET $2", + values: [count,offset,user] + }; return db.query(query) .then((result: pg.QueryResult) => result.rows); }; -export async function getRequestsVoted(count: number, user: number) { - var query = Object.assign(queries.getRequestsVoted, { values: [count,user] }); +export async function getRequestsTotal() { + return db.query(queries.getRequestsTotal) + .then((result: pg.QueryResult) => result.rows[0]["count"]); +}; + +export async function getAllRequests(count: number, offset: number, sort: string) { + var query = { + text: "SELECT * FROM requests_vw \ + ORDER BY " + sort + " LIMIT $1 OFFSET $2", + values: [count,offset] + }; return db.query(query) .then((result: pg.QueryResult) => result.rows); }; -export async function getAllRequestsVoted(count: number,user: number) { - var query = Object.assign(queries.getAllRequestsVoted, { values: [count,user] }); +export async function getAllRequestsVoted(count: number, offset: number, sort: string, user: number) { + var query = { + text: "SELECT * FROM get_requests_voted($3) \ + ORDER BY " + sort + " LIMIT $1 OFFSET $2", + values: [count,offset,user] + }; return db.query(query) .then((result: pg.QueryResult) => result.rows); }; +export async function getAllRequestsTotal() { + return db.query(queries.getAllRequestsTotal) + .then((result: pg.QueryResult) => result.rows[0]["count"]); +}; + const validUrlRegexes = [ /^https:\/\/www\.youtube\.com\/watch\?v=[a-zA-Z0-9_-]{11}$/ ]; @@ -59,6 +87,7 @@ async function retrieveYoutubeMetadata(url: string) { } export async function addRequest(url: string, requester: string): Promise<[number,string]> { + // Check that URL is of an accepted format var validUrl = false; for (var regex of validUrlRegexes) { if (regex.test(url)) { @@ -67,10 +96,24 @@ export async function addRequest(url: string, requester: string): Promise<[numbe } } if (!validUrl) return [400, "Invalid song URL."]; - var result = await checkRequestExists(url) - if (result) { - return [200,`Song already requested by ${result.rows[0].requester}. State: ${result.rows[0].state}`] + + // Check whether the URL has already been requested + var existsResult = await checkRequestExists(url) + if (existsResult) { + console.log(existsResult); + return [200,`Song already requested by ${existsResult.rows[0].requester}. State: ${existsResult.rows[0].state}`] } + + // Check whether the user has hit their rate limit + var rateLimitQuery = Object.assign(queries.getRateLimitStatus, { values: [requester] }); + var rateLimitResult = (await db.query(rateLimitQuery)).rows[0]; + if (rateLimitResult.status) { + return [429,`You have reached your maximum of ${rateLimitResult.max} requests per day. + Please try again later.\n + Tip: Removing one of your requests from the past 24 hours by retracting your vote will allow you to replace it with another.`]; + } + + // Add the request var query = Object.assign(queries.addRequest, { values: [url,requester] }); return db.query(query) .then(async () => { diff --git a/src/twitch.ts b/src/twitch.ts index e44b303..494bc1a 100644 --- a/src/twitch.ts +++ b/src/twitch.ts @@ -2,7 +2,6 @@ import * as config from "./config"; import * as queries from "./queries"; import { log, LogLevel } from "./logging" import fetch, { Response as FetchResponse } from "node-fetch"; -import pg from "pg"; import db from "./db"; export interface TokenPair { @@ -10,6 +9,11 @@ export interface TokenPair { refresh_token: string; } +export interface StreamerUserIdTokenPair { + userid: number + tokenpair: TokenPair +} + // Refresh the API token. Returns true on success and false on failure. async function refreshApiToken(tokens: TokenPair): Promise { log(LogLevel.DEBUG,`Call: twitch.refreshApiToken(${JSON.stringify(tokens,null,2)})`); @@ -25,8 +29,8 @@ async function refreshApiToken(tokens: TokenPair): Promise { if (res.status == 200) { log(LogLevel.INFO,"twitch.refreshApiToken: Refresh returned success."); var data = await (res.json() as Promise); - log(LogLevel.DEBUG, "Returned data:") - log(LogLevel.DEBUG, data) + log(LogLevel.DEBUG, "Returned data:"); + log(LogLevel.DEBUG, JSON.stringify(data,null,2)); tokens.access_token = data.access_token; tokens.refresh_token = data.refresh_token; return true; @@ -49,6 +53,7 @@ export async function apiRequest(tokens: TokenPair, endpoint: string): Promise < return fetch("https://api.twitch.tv/helix" + endpoint, { headers: headers }) .then(async (res: FetchResponse) => { if (res.status == 200) { + log(LogLevel.DEBUG,"twitch.apiRequest: Request returned 200 for " + endpoint); return res.json(); } else { log(LogLevel.WARNING,"twitch.apiRequest: Failed API request (pre-refresh):"); @@ -85,15 +90,15 @@ export async function apiRequest(tokens: TokenPair, endpoint: string): Promise < }) } -export async function streamerApiRequest(endpoint: string) { +export async function streamerApiRequest(streamer: StreamerUserIdTokenPair, endpoint: string) { log(LogLevel.DEBUG,`Call: twitch.streamerApiRequest(${endpoint})`); - var streamer = await db.query(queries.getStreamerIdToken).then((result: pg.QueryResult) => result.rows[0]); - var tokenpair = streamer.tokenpair; - var originaltoken = tokenpair.access_token; - log(LogLevel.DEBUG,"Original token: " + tokenpair.access_token); - var response = await apiRequest(tokenpair,endpoint); - log(LogLevel.DEBUG,"New token: " + tokenpair.access_token); - if (tokenpair.access_token != originaltoken) await db.query(Object.assign(queries.updateStreamer,{ values: [streamer.userid,tokenpair] })) + var originaltoken = streamer.tokenpair.access_token; + log(LogLevel.DEBUG,"Original token: " + originaltoken); + var response = await apiRequest(streamer.tokenpair,endpoint); + log(LogLevel.DEBUG,"New token: " + streamer.tokenpair.access_token); + if (streamer.tokenpair.access_token != originaltoken) + await db.query(Object.assign(queries.updateStreamer, + { values: [streamer.userid,streamer.tokenpair] })) return response; } diff --git a/src/version.ts b/src/version.ts index cfe9591..58b3b13 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,15 +1,30 @@ import * as queries from "./queries"; +import { log, LogLevel } from "./logging" +import express from "express"; import pg from "pg"; import db from "./db"; var versionMajor = 0; -var versionMinor = 2; -var versionPatch = 1; +var versionMinor = 8; +var versionPatch = 0; export function getVersion() { return `${versionMajor}.${versionMinor}.${versionPatch}` } +export async function checkVersionMiddleware(_req: express.Request, _res: express.Response, next: express.NextFunction) { + try { + await checkVersion(); + } catch (e) { + log(LogLevel.ERROR,e) + // Terminate the nodejs process with error. If restarted, the app will + // never start listening for new requests so this will not result in a + // restart loop. + process.exit(1); + } + next(); +} + export async function checkVersion() { var dbver = await db.query(queries.getDbVersion).then((result: pg.QueryResult) => result.rows[0]['get_version']); if (dbver != `${versionMajor}.${versionMinor}`) { diff --git a/views/colors.eta b/views/colors.eta new file mode 100644 index 0000000..b98c5da --- /dev/null +++ b/views/colors.eta @@ -0,0 +1,41 @@ +body { + background-color: <%= it.bg.primary %>; + color: <%= it.fg.primary %>; +} + +a { + color: <%= it.fg.primary %>; +} + +a:hover { + color: <%= it.fg.ahover %>; +} + +.error { + background-color: <%= it.bg.error %>; +} + +#navbar { + background-color: <%= it.bg.navbar %>; +} + +#nav-title, #nav-title a { + color: <%= it.fg.title %>; +} + +#main { + background-color: <%= it.bg.table %>; +} + +#modalBackground { + background-color: <%= it.bg.primary %>; + background-color: <%= it.bg.primary %>aa; +} + +.modal { + background-color: <%= it.bg.primary %>; +} + +#deleteRequestLink { + color: <%= it.bg.error %>; +} diff --git a/views/main.eta b/views/main.eta index 90f9f82..ce7d520 100644 --- a/views/main.eta +++ b/views/main.eta @@ -1,8 +1,9 @@ - - <%= it.streamerName %>'s Learn Request Queue + + + <%~ it.config.title.replace('{username}',it.streamerName) %>
Song TitleRequesterScoreState'; if (window.loggedIn) requestsDivHTML += 'Vote'; if (window.isStreamer) requestsDivHTML += 'Update
${request.title}${request.imageurl ? `` : ''}${request.requester}${request.score}