From 064857ecd673d8217766bed0c1d730aa34219f36 Mon Sep 17 00:00:00 2001 From: Baoyuan Date: Tue, 31 Aug 2021 21:24:28 +0800 Subject: [PATCH 1/4] feat: render svg in clould function --- api-go/tools/puppeteer/fetch.js | 123 ++++++++++++++++++++ api-go/tools/puppeteer/index.js | 109 +++++++++++------- api-go/tools/puppeteer/package.json | 7 +- api-go/tools/puppeteer/utils.js | 168 ++++++++++++++++++++++++++++ 4 files changed, 368 insertions(+), 39 deletions(-) create mode 100644 api-go/tools/puppeteer/fetch.js create mode 100644 api-go/tools/puppeteer/utils.js diff --git a/api-go/tools/puppeteer/fetch.js b/api-go/tools/puppeteer/fetch.js new file mode 100644 index 0000000..ede9244 --- /dev/null +++ b/api-go/tools/puppeteer/fetch.js @@ -0,0 +1,123 @@ +const moment = require('moment'); +const axios = require('axios'); + +const isSameDay = (d1, d2) => { + return ( + d1.getFullYear() === d2.getFullYear() && + d1.getMonth() === d2.getMonth() && + d1.getDate() === d2.getDate() + ); +}; + +const fetchContributorsData = (repo) => { + if (repo === "null" || repo === null) { + repo = "apache/apisix"; + } + return new Promise((resolve, reject) => { + axios.get( + `https://contributor-overtime-api.apiseven.com/contributors?repo=${repo}` + ).then(response => { + return response.data; + }).then(data => { + const { Contributors = [] } = data; + const sortContributors = Contributors.map(item => ({ + ...item, + date: item.date.substring(0, 10) + })).sort( + (a, b) => new Date(a.date).getTime() - new Date(b.date).getTime() + ); + if ( + !isSameDay( + new Date(sortContributors[sortContributors.length - 1].date), + new Date() + ) + ) { + sortContributors.push({ + repo, + idx: sortContributors[sortContributors.length - 1].idx, + date: moment(new Date()).format("YYYY-MM-DD") + }); + }; + + const processContributors = []; + sortContributors.forEach((item, index) => { + processContributors.push(item); + + if (index !== sortContributors.length - 1) { + const diffDays = moment(sortContributors[index + 1].date).diff( + item.date, + "days" + ); + if (diffDays > 1) { + for (let index = 1; index < diffDays; index++) { + processContributors.push({ + ...item, + date: moment(item.date) + .add(index, "days") + .format() + .substring(0, 10) + }); + } + } + } + }); + + const filterData = processContributors.filter( + (item, index) => + index === 0 || + index === processContributors.length - 1 || + new Date(item.date).getDate() % 10 === 5 + ); + + resolve({ repo, ...{ Contributors: filterData } }); + }).catch(error => { + reject(error); + }) + }) +}; + +const fetchMonthlyData = (repo) => { + if (repo === "null" || repo === null) { + repo = "apache/apisix"; + } + return new Promise((resolve, reject) => { + axios.get( + `https://contributor-overtime-api.apiseven.com/monthly-contributor?repo=${repo}` + ) + .then(response => { + return response.data; + }) + .then(myJson => { + resolve({ repo, ...myJson }); + }) + .catch(e => { + reject(e); + }); + }); +}; + +const fetchMergeContributor = (repo) => { + return new Promise((resolve, reject) => { + axios.get( + `https://contributor-overtime-api.apiseven.com/contributors-multi?repo=${repo.join( + "," + )}` + ) + .then(response => { + return response.data; + }) + .then(myJson => { + console.log('myJson: ', myJson); + resolve({ repo, ...myJson }); + }) + .catch(e => { + reject(e); + }); + }); +}; + +module.exports = { + fetchContributorsData, + fetchMonthlyData, + fetchMergeContributor, +} diff --git a/api-go/tools/puppeteer/index.js b/api-go/tools/puppeteer/index.js index 2159548..7e3c61a 100644 --- a/api-go/tools/puppeteer/index.js +++ b/api-go/tools/puppeteer/index.js @@ -4,51 +4,84 @@ * @param {!express:Request} req HTTP request context. * @param {!express:Response} res HTTP response context. */ -const puppeteer = require('puppeteer'); -const querystring = require('querystring'); +const echarts = require("echarts"); +const { createCanvas } = require('canvas'); +const jsdom = require("jsdom"); +const { fetchContributorsData, fetchMonthlyData } = require('./fetch'); +const { updateSeries } = require('./utils'); +const { JSDOM } = jsdom; + +const config = { + width: 896, + height: 550, +} exports.svg = async (req, res) => { const repo = req.query.repo; const merge = req.query.merge; - const chart = req.query.chart; - - const browser = await puppeteer.launch({ - args: ['--no-sandbox',] - }); - const page = await browser.newPage(); - await page.setViewport({ width: 1920, height: 1080 }); - - graphUrl = "https://contributor-graph-git-no-animation-apiseven.vercel.app/?repo=" + repo; - if (merge) { - graphUrl += "&merge=true" - } - if (chart) { - graphUrl += "&chart=" + chart - } + const chartType = req.query.chart; - await page.goto(graphUrl); + const window = (new JSDOM(` + + + + + + + Document + + +
+ + + `)).window; - var getSVG = function () { - return window.echartInstance.getDataURL(); - }; + const { document } = window; + global.document = document; - var svgReady = function () { - return window.echartsRenderFinished; - } + const ctx = createCanvas(config.width, config.height); + echarts.setCanvasCreator(function () { + return ctx; + }); + let repoList = repo.split(","); + Promise.all(repoList.map(item => { + if (chartType === "contributorMonthlyActivity") { + return fetchMonthlyData(item); + }; + return fetchContributorsData(item); + })).then(data => { + const tmpDataSouce = {}; + data.forEach(item => { + const { Contributors = [], repo } = item; + const data = Contributors.map(item => { + if (chartType === "contributorMonthlyActivity") { + return { + repo, + contributorNum: item.Num, + date: item.Month + } + } else { + return { + repo, + contributorNum: item.idx, + date: item.date + } + } + }); + + if (!tmpDataSouce[item.repo]) { + tmpDataSouce[repo] = data; + } + }); - function sleep(time) { - return new Promise((resolve) => setTimeout(resolve, time)); - } + const option = updateSeries(["1970-01-01"], tmpDataSouce); + let chart = echarts.init(document.getElementById('echarts'), {}, {renderer: "svg"}); + chart.setOption(option, true); + const image = chart.getSvgDataURL(); - sleep(1000).then(() => { - var _flagCheck = setInterval(async function () { - if (await page.evaluate(svgReady) === true) { - clearInterval(_flagCheck); - var image = await page.evaluate(getSVG); - res.set('Content-Type', 'image/svg+xml'); - res.send(querystring.unescape(image).split("data:image/svg+xml;charset=UTF-8,")[1]); - await browser.close(); - } - }, 100); - }); + res.set('Content-Type', 'image/svg+xml'); + res.send(decodeURIComponent(image).split("data:image/svg+xml;charset=UTF-8,")[1]); + }).catch(error => { + console.log(`generate file of ${repo} error`, error); + }) }; diff --git a/api-go/tools/puppeteer/package.json b/api-go/tools/puppeteer/package.json index c9987eb..1279c69 100644 --- a/api-go/tools/puppeteer/package.json +++ b/api-go/tools/puppeteer/package.json @@ -2,6 +2,11 @@ "name": "sample-http", "version": "0.0.1", "dependencies": { - "puppeteer": "8.0.0" + "axios": "^0.21.1", + "canvas": "^2.8.0", + "echarts": "5.1.2", + "lodash.clonedeep": "4.5.0", + "moment": "^2.29.1", + "jsdom": "^17.0.0" } } diff --git a/api-go/tools/puppeteer/utils.js b/api-go/tools/puppeteer/utils.js new file mode 100644 index 0000000..108bae9 --- /dev/null +++ b/api-go/tools/puppeteer/utils.js @@ -0,0 +1,168 @@ +const cloneDeep = require('lodash.clonedeep'); +const echarts = require("echarts"); + +const DEFAULT_COLOR = "#39a85a"; + +const generateDefaultOption = () => { + return { + color: ["#39a85a", "#4385ee", "#fabc37", "#2dc1dd", "#f972cf", "#8331c8"], + legend: { + top: "5%", + data: [], + textStyle: { + fontSize: 14 + } + }, + toolbox: { + feature: { + myShare: { + show: true, + title: "Share", + icon: "path://M830.506667 642.688c-57.344 0-104.192 30.72-126.037334 78.336l-277.162666-117.888c16.170667-25.045333 25.045333-55.722667 25.045333-88.832a167.253333 167.253333 0 0 0-4.864-40.362667l176.981333-137.301333c23.466667 17.749333 53.333333 28.245333 86.485334 28.245333 80 0 139.776-59.733333 139.776-138.88S790.912 87.04 710.954667 87.04c-80 0-139.008 59.733333-139.008 138.922667 0 20.181333 4.053333 38.741333 10.496 54.912L419.2 406.101333c-32.298667-48.469333-85.632-82.389333-146.218667-82.389333a178.944 178.944 0 0 0-179.413333 178.474667v0.853333a178.944 178.944 0 0 0 179.413333 179.2c37.12 0 71.893333-9.642667 100.181334-27.392l318.378666 135.68c4.053333 75.093333 62.250667 130.816 138.965334 130.816 80.042667 0 139.008-59.733333 139.008-138.922667 0-79.146667-59.818667-139.733333-138.965334-139.733333z", + onclick: function () { + handleShareClick(); + } + } + } + }, + dataset: [], + title: { + text: "Contributor Over Time" + }, + tooltip: { + trigger: "axis" + }, + xAxis: { + type: "time", + nameLocation: "middle", + axisLabel: { + show: true, + textStyle: { + fontSize: 14 + }, + rotate: 45 + } + }, + yAxis: { + name: "", + axisLabel: { + show: true, + textStyle: { + fontSize: 14 + } + } + }, + series: [], + grid: { + x: 10, + x2: 15, + y: 80, + containLabel: true, + y2: 5 + } + }; +}; + +const updateSeries = (passXAxis, dataSource) => { + const newClonedOption = cloneDeep( + generateDefaultOption() + ); + const datasetWithFilters = [ + ["ContributorNum", "Repo", "Date", "DateValue"] + ]; + const legend = []; + const limitDate = new Date(passXAxis[0]).getTime(); + Object.entries(dataSource).forEach(([key, value]) => { + legend.push(key); + value.forEach(item => { + datasetWithFilters.push([ + item.contributorNum, + item.repo, + item.date, + new Date(item.date).getTime() + ]); + }); + }); + + const newDateSet = datasetWithFilters.sort( + (a, b) => new Date(a[2]) - new Date(b[2]) + ); + + const filterDataset = legend.map(item => ({ + id: item, + fromDatasetId: "dataset_raw", + transform: { + type: "filter", + config: { + and: [ + { dimension: "Repo", "=": item }, + { dimension: "DateValue", gte: limitDate } + ] + } + } + })); + + const series = legend.map(item => ({ + name: item, + type: "line", + datasetId: item, + showSymbol: false, + smooth: true, + encode: { + x: "Date", + y: "ContributorNum", + itemName: "Repo", + tooltip: ["ContributorNum"] + } + })); + + if (series.length === 1) { + series[0].areaStyle = { + color: new echarts.graphic.LinearGradient(0, 0, 0, 1, [ + { + offset: 0, + color: DEFAULT_COLOR + "80" + }, + { + offset: 1, + color: DEFAULT_COLOR + "00" + } + ]) + }; + series[0].itemStyle = { + normal: { + color: DEFAULT_COLOR, + lineStyle: { + color: DEFAULT_COLOR + } + } + }; + } + + newClonedOption.dataset = [ + { + id: "dataset_raw", + source: newDateSet + } + ].concat(filterDataset); + + newClonedOption.series = series; + newClonedOption.legend.data = legend; + return newClonedOption; +}; + +const repoNameToFileName = (repo, merge, charType) => { + let fileName = repo; + if (merge) { + fileName = 'merge/' + fileName; + } + if (charType === 'contributorMonthlyActivity') { + fileName = "monthly/" + fileName; + } + return fileName + ".png"; +}; + +module.exports = { + updateSeries, + repoNameToFileName, +} From 34b36dbd9a9d5a57d50aa496a821f7d597772f0d Mon Sep 17 00:00:00 2001 From: Baoyuan Date: Wed, 1 Sep 2021 15:17:07 +0800 Subject: [PATCH 2/4] fix: compalte relate repo data --- api-go/tools/puppeteer/fetch.js | 4 +--- api-go/tools/puppeteer/index.js | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/api-go/tools/puppeteer/fetch.js b/api-go/tools/puppeteer/fetch.js index ede9244..9d9ac35 100644 --- a/api-go/tools/puppeteer/fetch.js +++ b/api-go/tools/puppeteer/fetch.js @@ -99,9 +99,7 @@ const fetchMonthlyData = (repo) => { const fetchMergeContributor = (repo) => { return new Promise((resolve, reject) => { axios.get( - `https://contributor-overtime-api.apiseven.com/contributors-multi?repo=${repo.join( - "," - )}` + `https://contributor-overtime-api.apiseven.com/contributors-multi?repo=${repo}` ) .then(response => { return response.data; diff --git a/api-go/tools/puppeteer/index.js b/api-go/tools/puppeteer/index.js index 7e3c61a..4c551b0 100644 --- a/api-go/tools/puppeteer/index.js +++ b/api-go/tools/puppeteer/index.js @@ -7,14 +7,21 @@ const echarts = require("echarts"); const { createCanvas } = require('canvas'); const jsdom = require("jsdom"); -const { fetchContributorsData, fetchMonthlyData } = require('./fetch'); +const { fetchContributorsData, fetchMonthlyData, fetchMergeContributor } = require('./fetch'); const { updateSeries } = require('./utils'); const { JSDOM } = jsdom; const config = { width: 896, height: 550, -} +}; + +const mergeRepoList = [ + "apache/apisix", + "apache/skywalking", + "apache/openwhisk", + "apache/dubbo" +]; exports.svg = async (req, res) => { const repo = req.query.repo; @@ -44,10 +51,17 @@ exports.svg = async (req, res) => { return ctx; }); let repoList = repo.split(","); + Promise.all(repoList.map(item => { if (chartType === "contributorMonthlyActivity") { + console.log(`render contributorMonthlyActivity for ${repo}`); return fetchMonthlyData(item); }; + if (merge === "true" && repoList.length === 1 && mergeRepoList.includes(repo)) { + console.log(`render merge contributor for ${repo}`); + return fetchMergeContributor(repo); + } + console.log(`render contributorData for ${repo}`); return fetchContributorsData(item); })).then(data => { const tmpDataSouce = {}; From f3f7d51a008dc3354dd73a412acb2d3c589945c5 Mon Sep 17 00:00:00 2001 From: Baoyuan Date: Wed, 1 Sep 2021 15:24:09 +0800 Subject: [PATCH 3/4] fix: delete useless functions --- api-go/tools/puppeteer/utils.js | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/api-go/tools/puppeteer/utils.js b/api-go/tools/puppeteer/utils.js index 108bae9..dd73d37 100644 --- a/api-go/tools/puppeteer/utils.js +++ b/api-go/tools/puppeteer/utils.js @@ -151,18 +151,6 @@ const updateSeries = (passXAxis, dataSource) => { return newClonedOption; }; -const repoNameToFileName = (repo, merge, charType) => { - let fileName = repo; - if (merge) { - fileName = 'merge/' + fileName; - } - if (charType === 'contributorMonthlyActivity') { - fileName = "monthly/" + fileName; - } - return fileName + ".png"; -}; - module.exports = { updateSeries, - repoNameToFileName, } From 32c59116a167016466f753391d0072ed9ad82a98 Mon Sep 17 00:00:00 2001 From: Baoyuan Date: Wed, 1 Sep 2021 16:12:53 +0800 Subject: [PATCH 4/4] fix title --- api-go/tools/puppeteer/index.js | 3 ++- api-go/tools/puppeteer/utils.js | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/api-go/tools/puppeteer/index.js b/api-go/tools/puppeteer/index.js index 4c551b0..a1549a5 100644 --- a/api-go/tools/puppeteer/index.js +++ b/api-go/tools/puppeteer/index.js @@ -88,7 +88,8 @@ exports.svg = async (req, res) => { } }); - const option = updateSeries(["1970-01-01"], tmpDataSouce); + const title = chartType === "contributorMonthlyActivity" ? "Monthly Active Contributors" : "Contributor Over Time"; + const option = updateSeries(["1970-01-01"], tmpDataSouce, title); let chart = echarts.init(document.getElementById('echarts'), {}, {renderer: "svg"}); chart.setOption(option, true); const image = chart.getSvgDataURL(); diff --git a/api-go/tools/puppeteer/utils.js b/api-go/tools/puppeteer/utils.js index dd73d37..ce5ee04 100644 --- a/api-go/tools/puppeteer/utils.js +++ b/api-go/tools/puppeteer/utils.js @@ -3,7 +3,7 @@ const echarts = require("echarts"); const DEFAULT_COLOR = "#39a85a"; -const generateDefaultOption = () => { +const generateDefaultOption = (title) => { return { color: ["#39a85a", "#4385ee", "#fabc37", "#2dc1dd", "#f972cf", "#8331c8"], legend: { @@ -27,7 +27,7 @@ const generateDefaultOption = () => { }, dataset: [], title: { - text: "Contributor Over Time" + text: title, }, tooltip: { trigger: "axis" @@ -63,9 +63,9 @@ const generateDefaultOption = () => { }; }; -const updateSeries = (passXAxis, dataSource) => { +const updateSeries = (passXAxis, dataSource, title) => { const newClonedOption = cloneDeep( - generateDefaultOption() + generateDefaultOption(title) ); const datasetWithFilters = [ ["ContributorNum", "Repo", "Date", "DateValue"]