diff --git a/api-go/tools/puppeteer/fetch.js b/api-go/tools/puppeteer/fetch.js new file mode 100644 index 0000000..9d9ac35 --- /dev/null +++ b/api-go/tools/puppeteer/fetch.js @@ -0,0 +1,121 @@ +const moment = require('moment'); +const axios = require('axios'); + +const isSameDay = (d1, d2) => { + return ( + d1.getFullYear() === d2.getFullYear() && + d1.getMonth() === d2.getMonth() && + d1.getDate() === d2.getDate() + ); +}; + +const fetchContributorsData = (repo) => { + if (repo === "null" || repo === null) { + repo = "apache/apisix"; + } + return new Promise((resolve, reject) => { + axios.get( + `https://contributor-overtime-api.apiseven.com/contributors?repo=${repo}` + ).then(response => { + return response.data; + }).then(data => { + const { Contributors = [] } = data; + const sortContributors = Contributors.map(item => ({ + ...item, + date: item.date.substring(0, 10) + })).sort( + (a, b) => new Date(a.date).getTime() - new Date(b.date).getTime() + ); + if ( + !isSameDay( + new Date(sortContributors[sortContributors.length - 1].date), + new Date() + ) + ) { + sortContributors.push({ + repo, + idx: sortContributors[sortContributors.length - 1].idx, + date: moment(new Date()).format("YYYY-MM-DD") + }); + }; + + const processContributors = []; + sortContributors.forEach((item, index) => { + processContributors.push(item); + + if (index !== sortContributors.length - 1) { + const diffDays = moment(sortContributors[index + 1].date).diff( + item.date, + "days" + ); + if (diffDays > 1) { + for (let index = 1; index < diffDays; index++) { + processContributors.push({ + ...item, + date: moment(item.date) + .add(index, "days") + .format() + .substring(0, 10) + }); + } + } + } + }); + + const filterData = processContributors.filter( + (item, index) => + index === 0 || + index === processContributors.length - 1 || + new Date(item.date).getDate() % 10 === 5 + ); + + resolve({ repo, ...{ Contributors: filterData } }); + }).catch(error => { + reject(error); + }) + }) +}; + +const fetchMonthlyData = (repo) => { + if (repo === "null" || repo === null) { + repo = "apache/apisix"; + } + return new Promise((resolve, reject) => { + axios.get( + `https://contributor-overtime-api.apiseven.com/monthly-contributor?repo=${repo}` + ) + .then(response => { + return response.data; + }) + .then(myJson => { + resolve({ repo, ...myJson }); + }) + .catch(e => { + reject(e); + }); + }); +}; + +const fetchMergeContributor = (repo) => { + return new Promise((resolve, reject) => { + axios.get( + `https://contributor-overtime-api.apiseven.com/contributors-multi?repo=${repo}` + ) + .then(response => { + return response.data; + }) + .then(myJson => { + console.log('myJson: ', myJson); + resolve({ repo, ...myJson }); + }) + .catch(e => { + reject(e); + }); + }); +}; + +module.exports = { + fetchContributorsData, + fetchMonthlyData, + fetchMergeContributor, +} diff --git a/api-go/tools/puppeteer/index.js b/api-go/tools/puppeteer/index.js index 2159548..a1549a5 100644 --- a/api-go/tools/puppeteer/index.js +++ b/api-go/tools/puppeteer/index.js @@ -4,51 +4,99 @@ * @param {!express:Request} req HTTP request context. * @param {!express:Response} res HTTP response context. */ -const puppeteer = require('puppeteer'); -const querystring = require('querystring'); +const echarts = require("echarts"); +const { createCanvas } = require('canvas'); +const jsdom = require("jsdom"); +const { fetchContributorsData, fetchMonthlyData, fetchMergeContributor } = require('./fetch'); +const { updateSeries } = require('./utils'); +const { JSDOM } = jsdom; + +const config = { + width: 896, + height: 550, +}; + +const mergeRepoList = [ + "apache/apisix", + "apache/skywalking", + "apache/openwhisk", + "apache/dubbo" +]; exports.svg = async (req, res) => { const repo = req.query.repo; const merge = req.query.merge; - const chart = req.query.chart; + const chartType = req.query.chart; + + const window = (new JSDOM(` + + + + + + + Document + + +
+ + + `)).window; + + const { document } = window; + global.document = document; - const browser = await puppeteer.launch({ - args: ['--no-sandbox',] + const ctx = createCanvas(config.width, config.height); + echarts.setCanvasCreator(function () { + return ctx; }); - const page = await browser.newPage(); - await page.setViewport({ width: 1920, height: 1080 }); - - graphUrl = "https://contributor-graph-git-no-animation-apiseven.vercel.app/?repo=" + repo; - if (merge) { - graphUrl += "&merge=true" - } - if (chart) { - graphUrl += "&chart=" + chart - } - - await page.goto(graphUrl); - - var getSVG = function () { - return window.echartInstance.getDataURL(); - }; - - var svgReady = function () { - return window.echartsRenderFinished; - } - - function sleep(time) { - return new Promise((resolve) => setTimeout(resolve, time)); - } - - sleep(1000).then(() => { - var _flagCheck = setInterval(async function () { - if (await page.evaluate(svgReady) === true) { - clearInterval(_flagCheck); - var image = await page.evaluate(getSVG); - res.set('Content-Type', 'image/svg+xml'); - res.send(querystring.unescape(image).split("data:image/svg+xml;charset=UTF-8,")[1]); - await browser.close(); + let repoList = repo.split(","); + + Promise.all(repoList.map(item => { + if (chartType === "contributorMonthlyActivity") { + console.log(`render contributorMonthlyActivity for ${repo}`); + return fetchMonthlyData(item); + }; + if (merge === "true" && repoList.length === 1 && mergeRepoList.includes(repo)) { + console.log(`render merge contributor for ${repo}`); + return fetchMergeContributor(repo); + } + console.log(`render contributorData for ${repo}`); + return fetchContributorsData(item); + })).then(data => { + const tmpDataSouce = {}; + data.forEach(item => { + const { Contributors = [], repo } = item; + const data = Contributors.map(item => { + if (chartType === "contributorMonthlyActivity") { + return { + repo, + contributorNum: item.Num, + date: item.Month + } + } else { + return { + repo, + contributorNum: item.idx, + date: item.date + } + } + }); + + if (!tmpDataSouce[item.repo]) { + tmpDataSouce[repo] = data; } - }, 100); - }); + }); + + const title = chartType === "contributorMonthlyActivity" ? "Monthly Active Contributors" : "Contributor Over Time"; + const option = updateSeries(["1970-01-01"], tmpDataSouce, title); + let chart = echarts.init(document.getElementById('echarts'), {}, {renderer: "svg"}); + chart.setOption(option, true); + const image = chart.getSvgDataURL(); + + res.set('Content-Type', 'image/svg+xml'); + res.send(decodeURIComponent(image).split("data:image/svg+xml;charset=UTF-8,")[1]); + }).catch(error => { + console.log(`generate file of ${repo} error`, error); + }) }; diff --git a/api-go/tools/puppeteer/package.json b/api-go/tools/puppeteer/package.json index c9987eb..1279c69 100644 --- a/api-go/tools/puppeteer/package.json +++ b/api-go/tools/puppeteer/package.json @@ -2,6 +2,11 @@ "name": "sample-http", "version": "0.0.1", "dependencies": { - "puppeteer": "8.0.0" + "axios": "^0.21.1", + "canvas": "^2.8.0", + "echarts": "5.1.2", + "lodash.clonedeep": "4.5.0", + "moment": "^2.29.1", + "jsdom": "^17.0.0" } } diff --git a/api-go/tools/puppeteer/utils.js b/api-go/tools/puppeteer/utils.js new file mode 100644 index 0000000..ce5ee04 --- /dev/null +++ b/api-go/tools/puppeteer/utils.js @@ -0,0 +1,156 @@ +const cloneDeep = require('lodash.clonedeep'); +const echarts = require("echarts"); + +const DEFAULT_COLOR = "#39a85a"; + +const generateDefaultOption = (title) => { + return { + color: ["#39a85a", "#4385ee", "#fabc37", "#2dc1dd", "#f972cf", "#8331c8"], + legend: { + top: "5%", + data: [], + textStyle: { + fontSize: 14 + } + }, + toolbox: { + feature: { + myShare: { + show: true, + title: "Share", + icon: "path://M830.506667 642.688c-57.344 0-104.192 30.72-126.037334 78.336l-277.162666-117.888c16.170667-25.045333 25.045333-55.722667 25.045333-88.832a167.253333 167.253333 0 0 0-4.864-40.362667l176.981333-137.301333c23.466667 17.749333 53.333333 28.245333 86.485334 28.245333 80 0 139.776-59.733333 139.776-138.88S790.912 87.04 710.954667 87.04c-80 0-139.008 59.733333-139.008 138.922667 0 20.181333 4.053333 38.741333 10.496 54.912L419.2 406.101333c-32.298667-48.469333-85.632-82.389333-146.218667-82.389333a178.944 178.944 0 0 0-179.413333 178.474667v0.853333a178.944 178.944 0 0 0 179.413333 179.2c37.12 0 71.893333-9.642667 100.181334-27.392l318.378666 135.68c4.053333 75.093333 62.250667 130.816 138.965334 130.816 80.042667 0 139.008-59.733333 139.008-138.922667 0-79.146667-59.818667-139.733333-138.965334-139.733333z", + onclick: function () { + handleShareClick(); + } + } + } + }, + dataset: [], + title: { + text: title, + }, + tooltip: { + trigger: "axis" + }, + xAxis: { + type: "time", + nameLocation: "middle", + axisLabel: { + show: true, + textStyle: { + fontSize: 14 + }, + rotate: 45 + } + }, + yAxis: { + name: "", + axisLabel: { + show: true, + textStyle: { + fontSize: 14 + } + } + }, + series: [], + grid: { + x: 10, + x2: 15, + y: 80, + containLabel: true, + y2: 5 + } + }; +}; + +const updateSeries = (passXAxis, dataSource, title) => { + const newClonedOption = cloneDeep( + generateDefaultOption(title) + ); + const datasetWithFilters = [ + ["ContributorNum", "Repo", "Date", "DateValue"] + ]; + const legend = []; + const limitDate = new Date(passXAxis[0]).getTime(); + Object.entries(dataSource).forEach(([key, value]) => { + legend.push(key); + value.forEach(item => { + datasetWithFilters.push([ + item.contributorNum, + item.repo, + item.date, + new Date(item.date).getTime() + ]); + }); + }); + + const newDateSet = datasetWithFilters.sort( + (a, b) => new Date(a[2]) - new Date(b[2]) + ); + + const filterDataset = legend.map(item => ({ + id: item, + fromDatasetId: "dataset_raw", + transform: { + type: "filter", + config: { + and: [ + { dimension: "Repo", "=": item }, + { dimension: "DateValue", gte: limitDate } + ] + } + } + })); + + const series = legend.map(item => ({ + name: item, + type: "line", + datasetId: item, + showSymbol: false, + smooth: true, + encode: { + x: "Date", + y: "ContributorNum", + itemName: "Repo", + tooltip: ["ContributorNum"] + } + })); + + if (series.length === 1) { + series[0].areaStyle = { + color: new echarts.graphic.LinearGradient(0, 0, 0, 1, [ + { + offset: 0, + color: DEFAULT_COLOR + "80" + }, + { + offset: 1, + color: DEFAULT_COLOR + "00" + } + ]) + }; + series[0].itemStyle = { + normal: { + color: DEFAULT_COLOR, + lineStyle: { + color: DEFAULT_COLOR + } + } + }; + } + + newClonedOption.dataset = [ + { + id: "dataset_raw", + source: newDateSet + } + ].concat(filterDataset); + + newClonedOption.series = series; + newClonedOption.legend.data = legend; + return newClonedOption; +}; + +module.exports = { + updateSeries, +}