From 173c22b90a64a5d2c35a4662e837d448bfe78cf2 Mon Sep 17 00:00:00 2001 From: Dessa Simpson Date: Mon, 9 Nov 2020 23:43:16 -0700 Subject: [PATCH] Implement fetching of video titles Fixes #2 --- db/21-requestMetadata.sql | 6 +++++ db/90-views.sql | 6 +++-- db/95-procedures.sql | 1 + public/main.js | 14 +++++++--- public/style.css | 2 +- src/app.ts | 37 +++++++++++++++++++++++++++ src/queries.ts | 10 ++++++++ src/requests.ts | 54 ++++++++++++++++++++++++++++++++++++--- src/youtube.ts | 4 +-- views/main.eta | 13 +++++++++- 10 files changed, 134 insertions(+), 13 deletions(-) create mode 100644 db/21-requestMetadata.sql diff --git a/db/21-requestMetadata.sql b/db/21-requestMetadata.sql new file mode 100644 index 0000000..5894457 --- /dev/null +++ b/db/21-requestMetadata.sql @@ -0,0 +1,6 @@ +CREATE TABLE requestMetadata ( + url varchar NOT NULL UNIQUE, + videoTitle varchar DEFAULT NULL, + PRIMARY KEY (url), + FOREIGN KEY (url) REFERENCES requests(url) ON DELETE CASCADE +); diff --git a/db/90-views.sql b/db/90-views.sql index e012690..e274069 100644 --- a/db/90-views.sql +++ b/db/90-views.sql @@ -1,5 +1,6 @@ CREATE OR REPLACE VIEW requests_vw AS - SELECT url,displayName AS requester,imageUrl,state,score,reqTimestamp FROM requests + SELECT url,COALESCE(videoTitle,url) AS title,displayName AS requester,imageUrl,state,score,reqTimestamp FROM requests + JOIN requestMetadata USING (url) JOIN scores USING (url) JOIN users ON requests.requester = users.userid ORDER BY score DESC, reqTimestamp ASC; @@ -7,6 +8,7 @@ CREATE OR REPLACE VIEW requests_vw AS CREATE OR REPLACE FUNCTION get_requests_voted(votinguserid int) RETURNS TABLE ( url varchar, + title varchar, requester varchar, imageUrl varchar, state varchar, @@ -16,7 +18,7 @@ CREATE OR REPLACE FUNCTION get_requests_voted(votinguserid int) ) LANGUAGE SQL AS $$ - SELECT url,requester,imageUrl,state,score,reqTimestamp, + SELECT url,title,requester,imageUrl,state,score,reqTimestamp, (CASE WHEN votes.userid IS NULL THEN FALSE ELSE TRUE END) AS voted FROM requests_vw LEFT JOIN votes ON (requests_vw.url = votes.requesturl AND votes.userid = votinguserid) diff --git a/db/95-procedures.sql b/db/95-procedures.sql index 84effd3..f2427bf 100644 --- a/db/95-procedures.sql +++ b/db/95-procedures.sql @@ -10,6 +10,7 @@ CREATE OR REPLACE PROCEDURE add_request(url varchar,requester int) LANGUAGE SQL AS $$ INSERT INTO requests (url,requester) VALUES (url,requester); + INSERT INTO requestMetadata (url) VALUES (url); INSERT INTO scores (url) VALUES (url); INSERT INTO votes (requesturl,userid) VALUES (url,requester); CALL update_scores(); diff --git a/public/main.js b/public/main.js index ef205c6..d4cf22c 100644 --- a/public/main.js +++ b/public/main.js @@ -17,13 +17,13 @@ function getRequests(count,allRequests) { } function buildTable() { - var requestsDivHTML = ''; + var requestsDivHTML = '
URLRequesterScore
'; requestsDivHTML += '' requestsDivHTML += ""; for (request of requests) { - requestsDivHTML += `\ + requestsDivHTML += `\ \ `; requestsDivHTML += ``; @@ -112,7 +112,7 @@ function closeAddRequestModal() { } function openUpdateRequestModal(tr) { - var url = tr.getElementsByClassName('request-url')[0].innerText; + var url = tr.getElementsByClassName('request-link')[0].firstChild.href; var score = tr.getElementsByClassName('request-score')[0].innerText; var state = tr.getElementsByClassName('request-state')[0].innerText; document.getElementById("updateRequestUrl").href = url; @@ -244,6 +244,14 @@ function updateRequestScoreModifier(url,scoreDiff) { }); } +function updateRequestMetadata(url) { + fetch("/api/updateRequestMetadata?url=" + url) + .then(response => { + updateTable(); + response.text().then(showMessage); + }); +} + function deleteRequest(url) { fetch("/api/deleteRequest", { method: 'POST', body: new URLSearchParams({ url: url diff --git a/public/style.css b/public/style.css index 75c6299..7f4cbba 100644 --- a/public/style.css +++ b/public/style.css @@ -172,7 +172,7 @@ div#nav-userpic { text-align: right; } -#scoreModifierHelp { +.helptext { font-size: 75%; } diff --git a/src/app.ts b/src/app.ts index 0c75db8..e67ab26 100644 --- a/src/app.ts +++ b/src/app.ts @@ -129,6 +129,31 @@ app.post("/api/updateRequestState", async (request, response) => { .catch((e: any) => errorHandler(request,response,e)); }); +app.get("/api/updateRequestMetadata", async (request, response) => { + if (request.session) await validateApiToken(request.session); + if (!request.session || !request.session.user) { + response.status(401); + response.send("Session expired; please log in again"); + return; + } + var streamerid = await db.query(queries.getStreamerId).then((result: pg.QueryResult) => result.rows[0]['userid']); + if (request.session.user.id != streamerid) { + response.status(401); + response.send("You are not the streamer"); + return; + } + if (!request.query.url) { + response.status(400); + response.send("Missing url"); + return; + } + requests.updateRequestMetadata(request.query.url as string).then((val: [number,string]) => { + response.status(val[0]); + response.send(val[1]); + }) + .catch((e: any) => errorHandler(request,response,e)); +}); + app.post("/api/updateRequestScoreModifier", async (request, response) => { if (request.session) await validateApiToken(request.session); if (!request.session || !request.session.user) { @@ -349,6 +374,18 @@ async function processBannedUsers() { setTimeout(processBannedUsers,600000+Math.floor(Math.random()*600000)) +async function processEmptyMetadata() { + log(LogLevel.INFO,"processEmptyMetadata run at " + new Date().toISOString()); + var result = await db.query(queries.getRequestsWithEmptyMetadata); + for (var row of result.rows) { + log(LogLevel.DEBUG,"Processing empty metadata for request: " + row['url']); + requests.updateRequestMetadata(row['url']); + } + setTimeout(processEmptyMetadata,3600000+Math.floor(Math.random()*900000)) // Run every 1-1.25 hours to balance load +} + +processEmptyMetadata(); + // Check version then listen version.checkVersion().then(_ => app.listen(config.port, () => { console.log(`Listening on port ${config.port}`); diff --git a/src/queries.ts b/src/queries.ts index bf6c28b..aefab8b 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -77,6 +77,16 @@ export const updateRequestState = { text: "UPDATE requests SET state = $2 WHERE url = $1" } +export const getRequestsWithEmptyMetadata = { + name: "getRequestsWithEmptyMetadata", + text: "SELECT url FROM requestMetadata WHERE videoTitle IS NULL" +} + +export const updateRequestMetadata = { + name: "updateRequestState", + text: "UPDATE requestMetadata SET videoTitle = $2 WHERE url = $1" +} + export const updateRequestScoreModifier = { name: "updateRequestScoreModifier", text: "CALL update_request_score_modifier($1,$2)" diff --git a/src/requests.ts b/src/requests.ts index ff3d982..d23611e 100644 --- a/src/requests.ts +++ b/src/requests.ts @@ -1,4 +1,6 @@ import * as queries from "./queries" +import * as youtube from "./youtube" +import { log, LogLevel } from "./logging" import pg from "pg"; import db from "./db"; @@ -30,6 +32,32 @@ const validUrlRegexes = [ /^https:\/\/www\.youtube\.com\/watch\?v=[a-zA-Z0-9_-]{11}$/ ]; +async function checkRequestExists(url: string) { + var query = Object.assign(queries.checkRequestExists, { values: [url] }); + var result = await db.query(query); + if (result.rowCount > 0) { + return result; + } else { + return false; + } +} + +async function retrieveYoutubeMetadata(url: string) { + var videoId = url.match(/^https:\/\/www\.youtube\.com\/watch\?v=([a-zA-Z0-9_-]{11})$/)![1]; + try { + var ytResponse = await youtube.apiRequest('/videos', new URLSearchParams({ id: videoId, part: 'snippet' })); + if (ytResponse) { + var title = ytResponse['items'][0]['snippet']['title']; + db.query(Object.assign(queries.updateRequestMetadata, { values: [url,title] })); + } + } catch(e) { + log(LogLevel.ERROR,"Failed to fetch YouTube metadata"); + log(LogLevel.ERROR,"Video ID: " + videoId); + log(LogLevel.ERROR,"Error info:"); + log(LogLevel.ERROR,e); + } +} + export async function addRequest(url: string, requester: string): Promise<[number,string]> { var validUrl = false; for (var regex of validUrlRegexes) { @@ -39,16 +67,30 @@ export async function addRequest(url: string, requester: string): Promise<[numbe } } if (!validUrl) return [400, "Invalid song URL."]; - var query = Object.assign(queries.checkRequestExists, { values: [url] }); - var result = await db.query(query); - if (result.rowCount > 0) { + var result = await checkRequestExists(url) + if (result) { return [200,`Song already requested by ${result.rows[0].requester}. State: ${result.rows[0].state}`] } var query = Object.assign(queries.addRequest, { values: [url,requester] }); return db.query(query) - .then(() => [201,"Song request added."] as [number,string]); + .then(async () => { + if (url.includes('youtube')){ + retrieveYoutubeMetadata(url); + } + return [201,"Song request added."] as [number,string]; + }) }; +export async function updateRequestMetadata(url: string): Promise <[number,string]> { + if (!checkRequestExists(url)) { + return [400,"Request does not exist."]; + } + if (url.includes('youtube')){ + retrieveYoutubeMetadata(url); + } + return [200,"Metadata update requested."]; +} + export async function updateRequestState(url: string, state: string): Promise<[number,string]> { var query = Object.assign(queries.checkValidState, { values: [state] }); var result = await db.query(query); @@ -56,6 +98,10 @@ export async function updateRequestState(url: string, state: string): Promise<[n return [400,"Invalid state"] } + if (!checkRequestExists(url)) { + return [400,"Request does not exist."]; + } + var query = Object.assign(queries.updateRequestState, { values: [url,state] }); return db.query(query) .then(() => [200,"Song request state updated."] as [number,string]); diff --git a/src/youtube.ts b/src/youtube.ts index a447f27..eaaadbc 100644 --- a/src/youtube.ts +++ b/src/youtube.ts @@ -3,9 +3,9 @@ import { log, LogLevel } from "./logging" import fetch, { Response as FetchResponse } from "node-fetch"; export async function apiRequest(endpoint: string, parameters: URLSearchParams): Promise { - log(LogLevel.DEBUG,`Call: youtube.apiRequest(${endpoint})`); + log(LogLevel.DEBUG,`Call: youtube.apiRequest(${endpoint},${parameters})`); parameters.set('key',config.youtubeSecret); - var requestUrl = "https://www.googleapis.com/youtube/v3/" + endpoint + "?" + parameters.toString(); + var requestUrl = "https://www.googleapis.com/youtube/v3" + endpoint + "?" + parameters.toString(); return fetch(requestUrl) .then(async (res: FetchResponse) => { if (res.status == 200) { diff --git a/views/main.eta b/views/main.eta index fb4afce..333af6f 100644 --- a/views/main.eta +++ b/views/main.eta @@ -75,13 +75,24 @@ document.getElementById('updateRequestUrl').innerText, document.getElementById('scoreModifierInput').value)">Submit
- + Enter a number to add to (or negative to subtract from) the score of a request. Use this for things like donations and channel points redemptions.
+
+ Update Request Metadata +
+ + Use this to update metadata about a request (i.e. title) if it + is missing (shows URL instead of name) or is outdated. + +
+
SongRequesterScoreState'; if (window.loggedIn) requestsDivHTML += 'Vote'; if (window.isStreamer) requestsDivHTML += 'Update
${request.url}
${request.title}${request.imageurl ? `` : ''}${request.requester}${request.score}${request.state}