feat: add mastodon

.github/workflows/fetch-tweets.yml

@@ -1,32 +1,32 @@
 name: Fetch Tweets and Publish new package
-on: 
+on:
     workflow_dispatch:
     schedule:
-      - cron:  '0 0 * * 6' 
+      - cron:  '0 0 * * 6'
 
 jobs:
   fetch:
     runs-on: ubuntu-latest
     steps:
     - name: Prep
-      uses: actions/setup-node@v1
+      uses: actions/setup-node@v3
       with:
-        node-version: '12'
+        node-version: '16'
         registry-url: 'https://npm.pkg.github.com'
         scope: 'codeimpossible'
     - name: Fetch Tweets
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - name: Fetch tweets and push
       env:
-        GITHUB_USER: 'codeimpossible'
         TWITTER_CONSUMER_KEY: ${{ secrets.TWITTER_CONSUMER_KEY }}
         TWITTER_CONSUMER_SECRET: ${{ secrets.TWITTER_CONSUMER_SECRET }}
         TWITTER_ACCESS_KEY: ${{ secrets.TWITTER_ACCESS_KEY }}
         TWITTER_ACCESS_TOKEN_SECRET: ${{ secrets.TWITTER_ACCESS_TOKEN_SECRET }}
+        MASTODON_TOKEN: ${{ secrets.MASTODON_TOKEN }}
         NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |
         yarn install
-        node ./bin/main.js ${GITHUB_USER}
+        node ./bin/main.js
         git config user.name github-actions
         git config user.email github-actions@github.com
         git add .

.github/workflows/publish-gpr.yml

@@ -7,11 +7,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       # Setup .npmrc file to publish to GitHub Packages
-      - uses: actions/setup-node@v1
+      - uses: actions/setup-node@v3
         with:
-          node-version: '12'
+          node-version: '16'
           registry-url: 'https://npm.pkg.github.com'
           scope: 'codeimpossible'
       # Publish to GitHub Packages

README.md

@@ -1,5 +1,5 @@
 # my-tweets
-My tweets, backed up and committed automatically via github actions.
+My tweets and mastodon posts, backed up and committed automatically via github actions.
 
 ## Loading my tweets
 
@@ -29,4 +29,4 @@ console.log(latest);
   ...
 */
 
-```
+```

bin/config.json

@@ -0,0 +1,9 @@
+{
+    "mastodon": {
+        "screen_name": "Literallyacat",
+        "host": "https://mastodon.gamedev.place"
+    },
+    "twitter": {
+        "screen_name": "codeimpossible"
+    }
+}

bin/main.js

@@ -1,44 +1,61 @@
 const fs = require('fs');
 const path = require('path');
-const { fetchTweets, setCredentials } = require('./../src/fetch-tweets');
 const { index, storeTweets } = require('./../src/db');
+const config = require('./config.json');
+
+const fetchTasks = [
+    require('./../src/fetch-tweets'),
+    require('./../src/fetch-toots'),
+];
 
 const args = process.argv.slice(2);
 
-if (args.length === 0) {
-    console.log('Usage: my-tweets twitter_user');
-    console.log(' twitter_user         The twitter username of the user to pull tweets for');
+if (args.includes('--help')) {
+    console.log('Usage: my-tweets');
+    console.log('    Fetches tweets and posts from twitter and mastodon and stores them in an indexed json catalog. Configuration is specified via a config.json file.');
+    console.log('');
+    console.log('  Configuration');
+    console.log('  {');
+    console.log('    "mastodon": {');
+    console.log('      "screen_name": "exampleUser"             // the screen name of the user to fetch posts for');
+    console.log('      "host": "http://mastodon.example.com"    // the hostname of the mastodon instance to interact with');
+    console.log('    },');
+    console.log('    "twitter": {');
+    console.log('      "screen_name": "exampleUser"             // the screen name of the user to fetch posts for');
+    console.log('    },');
+    console.log('  }');
     console.log('');
-    console.log('Example: $ my-tweets codeimpossible');
+    console.log('Example: $ my-tweets');
     process.exit(1);
 }
 
 process.on('uncaughtException', function (err) {
     if (err) {
-        console.log(`uncaught exception ${err}`, err.stack);
+        console.log(`uncaught exception ${err}`, err, err.stack);
+        console.log(JSON.stringify(err));
         process.exit(1);
     }
 });
 
-const credentialsFile = path.resolve(__dirname, '../credentials.json');
-if (fs.existsSync(credentialsFile)) {
-    const credentalsJson = fs.readFileSync(credentialsFile).toString();
-    const credentials = JSON.parse(credentalsJson);
-    console.log(credentials);
-    setCredentials(credentials);
-}
-
-(async function Main(screen_name, since) {
-    since = since || -1;
-    try {
-        const tweets = await fetchTweets(screen_name, since);
-        if (tweets.length > 0 && !tweets.errors) {
-            await storeTweets(tweets);
-            console.log(`🐣 stored ${tweets.length} tweets.`);
+(async function Main() {
+    for(const fetcher of fetchTasks) {
+        try {
+            const credentialsFile = path.resolve(__dirname, '../credentials.json');
+            if (fs.existsSync(credentialsFile)) {
+                const credentalsJson = fs.readFileSync(credentialsFile).toString();
+                const credentials = JSON.parse(credentalsJson);
+                console.log(credentials);
+                fetcher.setCredentials(credentials);
+            }
+            var data = await fetcher.fetch(config[fetcher.type], index);
+            if (data.length > 0) {
+                await storeTweets(data, fetcher.type);
+                console.log(`🐣 stored ${data.length} posts from ${fetcher.type}.`);
+            }
+        } catch (e) {
+            console.log(`exception while fetching from ${fetcher.type}.`, e, e.stack);
+            console.log(JSON.stringify(e));
         }
-    } catch (e) {
-        console.log(`exception while fetching tweets.`, e, e.stack);
-        console.log(JSON.stringify(e));
     }
     process.exit(0);
-})(args[0], index.latestId);
+})();

docs/modules.md

@@ -2,6 +2,7 @@
 
 | Module | Description |
 |--------|-------------|
-| `fetch-tweets` | Responsible for getting the latest tweets. Can be given a tweet id to use as a filter, returning only the tweets that have been posted after that id. |
+| `fetch-toots` | Responsible for getting the latest posts from a Mastodon instance. Will fetch posts older than the latest post id from the passed index database. |
+| `fetch-tweets` | Responsible for getting the latest tweets. Will fetch posts older than the latest post id from the passed index database. |
 | `db` | Used whenever you want to read/write tweets from/to storage. Has some helper methods for filtering/sorting. |
-| `main` | The main script. Currently loads the database index, queries twitter api for all tweets after the latest tweet and stores the results to the db. |
+| `main` | The main script. Currently loads the database index, queries twitter api for all tweets after the latest tweet and stores the results to the db. |

package.json

@@ -16,6 +16,7 @@
   ],
   "license": "MIT",
   "dependencies": {
+    "axios": "^1.4.0",
     "md5": "^2.3.0",
     "oauth": "^0.9.15"
   }

src/db/index.js

@@ -61,13 +61,13 @@ module.exports.getTweets = loadTweets;
 module.exports.sorts = sorts;
 module.exports.selectors = selectors;
 
-module.exports.storeTweets = async function(tweets) {
+module.exports.storeTweets = async function(toots, tootType) {
     // find the latest id in the collections
-    const latestTweet = selectors.first(tweets.sort(sorts.byDateDesc));
-    const json = JSON.stringify(tweets);
+    const latestTweet = selectors.first(toots.sort(sorts.byDateDesc));
+    const json = JSON.stringify(toots);
     const id = md5(json);
-    await writeJson(path.resolve(data_dir, `${id}.json`), tweets);
-    idx.latestId = latestTweet.id_str;
+    await writeJson(path.resolve(data_dir, `${id}.json`), toots);
+    idx[`latest_${tootType}_id`] = latestTweet.id;
     idx.sources.push(`${id}.json`);
     await saveIndex();
-};
+};

src/fetch-toots/index.js

@@ -0,0 +1,45 @@
+const axios = require('axios');
+const { MASTODON } = require('../tootTypes');
+
+let credentials = {
+    mastodonToken: process.env.MASTODON_USER_TOKEN,
+};
+
+module.exports.setCredentials = function(newCredentials) {
+    credentials = newCredentials;
+};
+
+module.exports.type = MASTODON;
+
+module.exports.fetch = async function(config, idx) {
+    const { mastodonToken } = credentials;
+    const { screen_name, host } = config;
+    const since_id = idx.latest_mastodon_id || -1;
+    // get the id of the user
+    const idApiUri = `${host}/api/v1/accounts/lookup?acct=@${screen_name}`;
+    const userInfoResponse = await axios.get(idApiUri, {
+        headers: {
+            'Authorization': `Bearer ${mastodonToken}`
+        }
+    });
+    console.log(userInfoResponse.data);
+    const { id } = userInfoResponse.data;
+
+    // get the toots
+    const tootsApiUri = `${host}/api/v1/accounts/${id}/statuses`;
+    const tootsResponse = await axios.get(tootsApiUri, {
+        params: {
+            since_id
+        },
+        headers: {
+            'Authorization': `Bearer ${mastodonToken}`
+        }
+    });
+    let toots = tootsResponse.data;
+    toots = toots.map(toot => {
+        toot.toot_type = MASTODON;
+        return toot;
+    });
+
+    return toots;
+};

src/fetch-tweets/index.js

@@ -1,5 +1,6 @@
 const { promisify } = require('util');
 const { OAuth } = require('oauth');
+const { TWITTER, MASTODON } = require('../tootTypes');
 
 let credentials = {
     consumerKey: process.env.TWITTER_CONSUMER_KEY,
@@ -12,7 +13,11 @@ module.exports.setCredentials = function(newCredentials) {
     credentials = newCredentials;
 }
 
-module.exports.fetchTweets = async function(screen_name, since_id=-1) {
+module.exports.type = TWITTER;
+
+module.exports.fetch = async function(config, idx) {
+    since_id = idx.latest_twitter_id || idx.latestId;
+    const { screen_name } = config;
     const oauth = new OAuth(
         'https://api.twitter.com/oauth/request_token',
         'https://api.twitter.com/oauth/access_token',
@@ -21,16 +26,20 @@ module.exports.fetchTweets = async function(screen_name, since_id=-1) {
         '1.0A', null, 'HMAC-SHA1'
     );
     const get = promisify(oauth.get.bind(oauth));
-    let params = `screen_name=${screen_name}`;
-    if (since_id > -1) {
-        params += `&since_id=${since_id}`;
-    }
-
     const body = await get(
-        `https://api.twitter.com/1.1/statuses/user_timeline.json?${params}`,
+        `https://api.twitter.com/2/users/${screen_name}/tweets?since_id=${since_id}`,
         credentials.accessKey,
         credentials.accessSecret
     );
 
-    return JSON.parse(body);
-};
+    const tweets = JSON.parse(body);
+
+    tweets = tweets.map(toot => {
+        toot.toot_type = TWITTER;
+        toot.id_num = toot.id;
+        toot.id = toot.id_str;
+        return toot;
+    });
+
+    return tweets;
+};

src/tootTypes.js

@@ -0,0 +1,4 @@
+module.exports = {
+    MASTODON: 'mastodon',
+    TWITTER: 'twitter',
+};

yarn.lock

@@ -2,16 +2,56 @@
 # yarn lockfile v1
 
 
+asynckit@^0.4.0:
+  version "0.4.0"
+  resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
+  integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==
+
+axios@^1.4.0:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-1.4.0.tgz#38a7bf1224cd308de271146038b551d725f0be1f"
+  integrity sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==
+  dependencies:
+    follow-redirects "^1.15.0"
+    form-data "^4.0.0"
+    proxy-from-env "^1.1.0"
+
 charenc@0.0.2:
   version "0.0.2"
   resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667"
   integrity sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=
 
+combined-stream@^1.0.8:
+  version "1.0.8"
+  resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f"
+  integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==
+  dependencies:
+    delayed-stream "~1.0.0"
+
 crypt@0.0.2:
   version "0.0.2"
   resolved "https://registry.yarnpkg.com/crypt/-/crypt-0.0.2.tgz#88d7ff7ec0dfb86f713dc87bbb42d044d3e6c41b"
   integrity sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=
 
+delayed-stream@~1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619"
+  integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==
+
+follow-redirects@^1.15.0:
+  version "1.15.2"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13"
+  integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==
+
+form-data@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452"
+  integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==
+  dependencies:
+    asynckit "^0.4.0"
+    combined-stream "^1.0.8"
+    mime-types "^2.1.12"
+
 is-buffer@~1.1.6:
   version "1.1.6"
   resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be"
@@ -26,7 +66,24 @@ md5@^2.3.0:
     crypt "0.0.2"
     is-buffer "~1.1.6"
 
+mime-db@1.52.0:
+  version "1.52.0"
+  resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70"
+  integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==
+
+mime-types@^2.1.12:
+  version "2.1.35"
+  resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a"
+  integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==
+  dependencies:
+    mime-db "1.52.0"
+
 oauth@^0.9.15:
   version "0.9.15"
   resolved "https://registry.yarnpkg.com/oauth/-/oauth-0.9.15.tgz#bd1fefaf686c96b75475aed5196412ff60cfb9c1"
   integrity sha1-vR/vr2hslrdUda7VGWQS/2DPucE=
+
+proxy-from-env@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2"
+  integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==