From 5198b0fc8b887e7a453f4f87cbfbe9d07f1359a0 Mon Sep 17 00:00:00 2001 From: Jared Barboza Date: Sat, 15 Jul 2023 07:53:29 -0400 Subject: [PATCH 1/2] chore: upgrade github actions --- .github/workflows/fetch-tweets.yml | 10 +++++----- .github/workflows/publish-gpr.yml | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/fetch-tweets.yml b/.github/workflows/fetch-tweets.yml index d4debcb..bbbca8a 100644 --- a/.github/workflows/fetch-tweets.yml +++ b/.github/workflows/fetch-tweets.yml @@ -1,21 +1,21 @@ name: Fetch Tweets and Publish new package -on: +on: workflow_dispatch: schedule: - - cron: '0 0 * * 6' + - cron: '0 0 * * 6' jobs: fetch: runs-on: ubuntu-latest steps: - name: Prep - uses: actions/setup-node@v1 + uses: actions/setup-node@v3 with: - node-version: '12' + node-version: '16' registry-url: 'https://npm.pkg.github.com' scope: 'codeimpossible' - name: Fetch Tweets - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Fetch tweets and push env: GITHUB_USER: 'codeimpossible' diff --git a/.github/workflows/publish-gpr.yml b/.github/workflows/publish-gpr.yml index c489e00..ccdeac3 100644 --- a/.github/workflows/publish-gpr.yml +++ b/.github/workflows/publish-gpr.yml @@ -7,11 +7,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 # Setup .npmrc file to publish to GitHub Packages - - uses: actions/setup-node@v1 + - uses: actions/setup-node@v3 with: - node-version: '12' + node-version: '16' registry-url: 'https://npm.pkg.github.com' scope: 'codeimpossible' # Publish to GitHub Packages From 2b5445d50ecf2cc1ec15ba238be87a7264cccf1b Mon Sep 17 00:00:00 2001 From: Jared Barboza Date: Sat, 15 Jul 2023 09:35:41 -0400 Subject: [PATCH 2/2] feat: fetch mastodon toots --- .github/workflows/fetch-tweets.yml | 4 +- README.md | 4 +- bin/config.json | 9 ++++ bin/main.js | 67 +++++++++++++++++++----------- docs/modules.md | 5 ++- package.json | 1 + src/db/index.js | 12 +++--- src/fetch-toots/index.js | 45 ++++++++++++++++++++ src/fetch-tweets/index.js | 27 ++++++++---- src/tootTypes.js | 4 ++ yarn.lock | 57 +++++++++++++++++++++++++ 11 files changed, 189 insertions(+), 46 deletions(-) create mode 100644 bin/config.json create mode 100644 src/fetch-toots/index.js create mode 100644 src/tootTypes.js diff --git a/.github/workflows/fetch-tweets.yml b/.github/workflows/fetch-tweets.yml index bbbca8a..b4ad5ff 100644 --- a/.github/workflows/fetch-tweets.yml +++ b/.github/workflows/fetch-tweets.yml @@ -18,15 +18,15 @@ jobs: uses: actions/checkout@v3 - name: Fetch tweets and push env: - GITHUB_USER: 'codeimpossible' TWITTER_CONSUMER_KEY: ${{ secrets.TWITTER_CONSUMER_KEY }} TWITTER_CONSUMER_SECRET: ${{ secrets.TWITTER_CONSUMER_SECRET }} TWITTER_ACCESS_KEY: ${{ secrets.TWITTER_ACCESS_KEY }} TWITTER_ACCESS_TOKEN_SECRET: ${{ secrets.TWITTER_ACCESS_TOKEN_SECRET }} + MASTODON_TOKEN: ${{ secrets.MASTODON_TOKEN }} NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | yarn install - node ./bin/main.js ${GITHUB_USER} + node ./bin/main.js git config user.name github-actions git config user.email github-actions@github.com git add . diff --git a/README.md b/README.md index 4ec81a8..2b26704 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # my-tweets -My tweets, backed up and committed automatically via github actions. +My tweets and mastodon posts, backed up and committed automatically via github actions. ## Loading my tweets @@ -29,4 +29,4 @@ console.log(latest); ... */ -``` \ No newline at end of file +``` diff --git a/bin/config.json b/bin/config.json new file mode 100644 index 0000000..7d845ce --- /dev/null +++ b/bin/config.json @@ -0,0 +1,9 @@ +{ + "mastodon": { + "screen_name": "Literallyacat", + "host": "https://mastodon.gamedev.place" + }, + "twitter": { + "screen_name": "codeimpossible" + } +} diff --git a/bin/main.js b/bin/main.js index bac8056..bc519cc 100644 --- a/bin/main.js +++ b/bin/main.js @@ -1,44 +1,61 @@ const fs = require('fs'); const path = require('path'); -const { fetchTweets, setCredentials } = require('./../src/fetch-tweets'); const { index, storeTweets } = require('./../src/db'); +const config = require('./config.json'); + +const fetchTasks = [ + require('./../src/fetch-tweets'), + require('./../src/fetch-toots'), +]; const args = process.argv.slice(2); -if (args.length === 0) { - console.log('Usage: my-tweets twitter_user'); - console.log(' twitter_user The twitter username of the user to pull tweets for'); +if (args.includes('--help')) { + console.log('Usage: my-tweets'); + console.log(' Fetches tweets and posts from twitter and mastodon and stores them in an indexed json catalog. Configuration is specified via a config.json file.'); + console.log(''); + console.log(' Configuration'); + console.log(' {'); + console.log(' "mastodon": {'); + console.log(' "screen_name": "exampleUser" // the screen name of the user to fetch posts for'); + console.log(' "host": "http://mastodon.example.com" // the hostname of the mastodon instance to interact with'); + console.log(' },'); + console.log(' "twitter": {'); + console.log(' "screen_name": "exampleUser" // the screen name of the user to fetch posts for'); + console.log(' },'); + console.log(' }'); console.log(''); - console.log('Example: $ my-tweets codeimpossible'); + console.log('Example: $ my-tweets'); process.exit(1); } process.on('uncaughtException', function (err) { if (err) { - console.log(`uncaught exception ${err}`, err.stack); + console.log(`uncaught exception ${err}`, err, err.stack); + console.log(JSON.stringify(err)); process.exit(1); } }); -const credentialsFile = path.resolve(__dirname, '../credentials.json'); -if (fs.existsSync(credentialsFile)) { - const credentalsJson = fs.readFileSync(credentialsFile).toString(); - const credentials = JSON.parse(credentalsJson); - console.log(credentials); - setCredentials(credentials); -} - -(async function Main(screen_name, since) { - since = since || -1; - try { - const tweets = await fetchTweets(screen_name, since); - if (tweets.length > 0 && !tweets.errors) { - await storeTweets(tweets); - console.log(`🐣 stored ${tweets.length} tweets.`); +(async function Main() { + for(const fetcher of fetchTasks) { + try { + const credentialsFile = path.resolve(__dirname, '../credentials.json'); + if (fs.existsSync(credentialsFile)) { + const credentalsJson = fs.readFileSync(credentialsFile).toString(); + const credentials = JSON.parse(credentalsJson); + console.log(credentials); + fetcher.setCredentials(credentials); + } + var data = await fetcher.fetch(config[fetcher.type], index); + if (data.length > 0) { + await storeTweets(data, fetcher.type); + console.log(`🐣 stored ${data.length} posts from ${fetcher.type}.`); + } + } catch (e) { + console.log(`exception while fetching from ${fetcher.type}.`, e, e.stack); + console.log(JSON.stringify(e)); } - } catch (e) { - console.log(`exception while fetching tweets.`, e, e.stack); - console.log(JSON.stringify(e)); } process.exit(0); -})(args[0], index.latestId); +})(); diff --git a/docs/modules.md b/docs/modules.md index f9ffeef..fdea5b1 100644 --- a/docs/modules.md +++ b/docs/modules.md @@ -2,6 +2,7 @@ | Module | Description | |--------|-------------| -| `fetch-tweets` | Responsible for getting the latest tweets. Can be given a tweet id to use as a filter, returning only the tweets that have been posted after that id. | +| `fetch-toots` | Responsible for getting the latest posts from a Mastodon instance. Will fetch posts older than the latest post id from the passed index database. | +| `fetch-tweets` | Responsible for getting the latest tweets. Will fetch posts older than the latest post id from the passed index database. | | `db` | Used whenever you want to read/write tweets from/to storage. Has some helper methods for filtering/sorting. | -| `main` | The main script. Currently loads the database index, queries twitter api for all tweets after the latest tweet and stores the results to the db. | \ No newline at end of file +| `main` | The main script. Currently loads the database index, queries twitter api for all tweets after the latest tweet and stores the results to the db. | diff --git a/package.json b/package.json index 52000a0..23fb838 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ ], "license": "MIT", "dependencies": { + "axios": "^1.4.0", "md5": "^2.3.0", "oauth": "^0.9.15" } diff --git a/src/db/index.js b/src/db/index.js index 572d0bf..e926721 100644 --- a/src/db/index.js +++ b/src/db/index.js @@ -61,13 +61,13 @@ module.exports.getTweets = loadTweets; module.exports.sorts = sorts; module.exports.selectors = selectors; -module.exports.storeTweets = async function(tweets) { +module.exports.storeTweets = async function(toots, tootType) { // find the latest id in the collections - const latestTweet = selectors.first(tweets.sort(sorts.byDateDesc)); - const json = JSON.stringify(tweets); + const latestTweet = selectors.first(toots.sort(sorts.byDateDesc)); + const json = JSON.stringify(toots); const id = md5(json); - await writeJson(path.resolve(data_dir, `${id}.json`), tweets); - idx.latestId = latestTweet.id_str; + await writeJson(path.resolve(data_dir, `${id}.json`), toots); + idx[`latest_${tootType}_id`] = latestTweet.id; idx.sources.push(`${id}.json`); await saveIndex(); -}; \ No newline at end of file +}; diff --git a/src/fetch-toots/index.js b/src/fetch-toots/index.js new file mode 100644 index 0000000..308ad81 --- /dev/null +++ b/src/fetch-toots/index.js @@ -0,0 +1,45 @@ +const axios = require('axios'); +const { MASTODON } = require('../tootTypes'); + +let credentials = { + mastodonToken: process.env.MASTODON_USER_TOKEN, +}; + +module.exports.setCredentials = function(newCredentials) { + credentials = newCredentials; +}; + +module.exports.type = MASTODON; + +module.exports.fetch = async function(config, idx) { + const { mastodonToken } = credentials; + const { screen_name, host } = config; + const since_id = idx.latest_mastodon_id || -1; + // get the id of the user + const idApiUri = `${host}/api/v1/accounts/lookup?acct=@${screen_name}`; + const userInfoResponse = await axios.get(idApiUri, { + headers: { + 'Authorization': `Bearer ${mastodonToken}` + } + }); + console.log(userInfoResponse.data); + const { id } = userInfoResponse.data; + + // get the toots + const tootsApiUri = `${host}/api/v1/accounts/${id}/statuses`; + const tootsResponse = await axios.get(tootsApiUri, { + params: { + since_id + }, + headers: { + 'Authorization': `Bearer ${mastodonToken}` + } + }); + let toots = tootsResponse.data; + toots = toots.map(toot => { + toot.toot_type = MASTODON; + return toot; + }); + + return toots; +}; diff --git a/src/fetch-tweets/index.js b/src/fetch-tweets/index.js index 9aea9c3..f274779 100644 --- a/src/fetch-tweets/index.js +++ b/src/fetch-tweets/index.js @@ -1,5 +1,6 @@ const { promisify } = require('util'); const { OAuth } = require('oauth'); +const { TWITTER, MASTODON } = require('../tootTypes'); let credentials = { consumerKey: process.env.TWITTER_CONSUMER_KEY, @@ -12,7 +13,11 @@ module.exports.setCredentials = function(newCredentials) { credentials = newCredentials; } -module.exports.fetchTweets = async function(screen_name, since_id=-1) { +module.exports.type = TWITTER; + +module.exports.fetch = async function(config, idx) { + since_id = idx.latest_twitter_id || idx.latestId; + const { screen_name } = config; const oauth = new OAuth( 'https://api.twitter.com/oauth/request_token', 'https://api.twitter.com/oauth/access_token', @@ -21,16 +26,20 @@ module.exports.fetchTweets = async function(screen_name, since_id=-1) { '1.0A', null, 'HMAC-SHA1' ); const get = promisify(oauth.get.bind(oauth)); - let params = `screen_name=${screen_name}`; - if (since_id > -1) { - params += `&since_id=${since_id}`; - } - const body = await get( - `https://api.twitter.com/1.1/statuses/user_timeline.json?${params}`, + `https://api.twitter.com/2/users/${screen_name}/tweets?since_id=${since_id}`, credentials.accessKey, credentials.accessSecret ); - return JSON.parse(body); -}; \ No newline at end of file + const tweets = JSON.parse(body); + + tweets = tweets.map(toot => { + toot.toot_type = TWITTER; + toot.id_num = toot.id; + toot.id = toot.id_str; + return toot; + }); + + return tweets; +}; diff --git a/src/tootTypes.js b/src/tootTypes.js new file mode 100644 index 0000000..33a3768 --- /dev/null +++ b/src/tootTypes.js @@ -0,0 +1,4 @@ +module.exports = { + MASTODON: 'mastodon', + TWITTER: 'twitter', +}; diff --git a/yarn.lock b/yarn.lock index e158dba..9007a4f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,16 +2,56 @@ # yarn lockfile v1 +asynckit@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" + integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== + +axios@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.4.0.tgz#38a7bf1224cd308de271146038b551d725f0be1f" + integrity sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA== + dependencies: + follow-redirects "^1.15.0" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + charenc@0.0.2: version "0.0.2" resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667" integrity sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc= +combined-stream@^1.0.8: + version "1.0.8" + resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" + integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== + dependencies: + delayed-stream "~1.0.0" + crypt@0.0.2: version "0.0.2" resolved "https://registry.yarnpkg.com/crypt/-/crypt-0.0.2.tgz#88d7ff7ec0dfb86f713dc87bbb42d044d3e6c41b" integrity sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs= +delayed-stream@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" + integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== + +follow-redirects@^1.15.0: + version "1.15.2" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13" + integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA== + +form-data@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" + integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + is-buffer@~1.1.6: version "1.1.6" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" @@ -26,7 +66,24 @@ md5@^2.3.0: crypt "0.0.2" is-buffer "~1.1.6" +mime-db@1.52.0: + version "1.52.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" + integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== + +mime-types@^2.1.12: + version "2.1.35" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + oauth@^0.9.15: version "0.9.15" resolved "https://registry.yarnpkg.com/oauth/-/oauth-0.9.15.tgz#bd1fefaf686c96b75475aed5196412ff60cfb9c1" integrity sha1-vR/vr2hslrdUda7VGWQS/2DPucE= + +proxy-from-env@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" + integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==