Paste Search Dynamic
Recent pastes
node-fetch
  1. const fs = require('fs');
  2. const fetch = require('node-fetch');
  3.  
  4. function escapeQuotes(str = "") {
  5.     return str.replace(/'/g, "''");
  6. }
  7.  
  8. async function fetchDisqusComments(firstURL) {
  9.         return new Promise(async (resolve, reject) => {
  10.                 let isDone = false;
  11.  
  12.                 let posts = [];
  13.                 let curURL = firstURL;
  14.                 let page = 1;
  15.                 while (!isDone) {
  16.                         console.log(`Fetching page ${page}`);
  17.                         const response = await fetch(curURL);
  18.                         const json = await response.json();
  19.                         console.log(`Success fetching page ${page}`);
  20.  
  21.                         posts.push(json.response);
  22.  
  23.                         if (json.cursor.hasNext) {
  24.                                 curURL = `${firstURL}&cursor=${json.cursor.next}`;
  25.                         } else {
  26.                                 isDone = true;
  27.                         }
  28.  
  29.                         page++;
  30.                 }
  31.  
  32.                 resolve(posts);
  33.         });
  34. }
  35.  
  36. /**
  37. * Converts Disqus comments array into an SQL transaction statement to insert into Commento's
  38.  * Postgres database.
  39.  */
  40. function commentsToSQL(comments) {
  41.     let sql = comments.reduce((accum, comment) => {
  42.         const name = escapeQuotes(comment.author.name);
  43.  
  44.         // Add avatar to user if found (Commento calls this the "photo").
  45.         const hasAvatar = comment.author.avatar.large.permalink.match(/images\/noavatar/) === null;
  46.         if (hasAvatar) {
  47.             accum = accum.concat(`
  48. UPDATE commenters
  49. SET photo='${comment.author.avatar.permalink}'
  50. WHERE
  51.     name='${name}'
  52.     AND photo='undefined';`);
  53.         }
  54.  
  55.         // Add link to author's website, or their Disqus profile url.
  56.         const link = comment.author.url || comment.author.profileUrl;
  57.         accum = accum.concat(`
  58. UPDATE commenters
  59. SET link='${link}'
  60. WHERE
  61.     name='${name}'
  62.     AND link='undefined';`);
  63.  
  64.         // Adds post points (Commento calls this the "score").
  65.         if (comment.points > 0) {
  66.         accum = accum.concat(`
  67. UPDATE comments
  68. SET score=${comment.points}
  69. WHERE
  70.     score=0
  71.     AND html LIKE '%${escapeQuotes(comment.message.substr(0, 50))}%'
  72.     AND exists (
  73.         SELECT *
  74.         FROM commenters
  75.         WHERE commenters.commenterhex = comments.commenterhex
  76.         AND commenters.name = '${name}'
  77.     );
  78. `);
  79.         }
  80.  
  81.  
  82.       return accum;
  83.     }, 'BEGIN;');
  84.  
  85.     sql = `${sql}\nCOMMIT;`;
  86.  
  87.     return sql;
  88. }
  89.  
  90. async function init(firstURL, filePath = '/tmp/disqus-comments.sql') {
  91.     const comments = await fetchDisqusComments(firstURL);
  92.  
  93.     const sql = commentsToSQL(comments);
  94.  
  95.     fs.writeFile(filePath, sql, function(err) {
  96.         if(err) {
  97.             return console.log(err);
  98.         }
  99.         console.log(`Comments saved to ${filePath}`);
  100.     });
  101. }
  102.  
  103. init('https://disqus.com/api/3.0/posts/list?attach=...');
  104.  
Parsed in 0.015 seconds