diff --git a/.gitignore b/.gitignore index 0c45ee7..c8fb3cf 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ /node_modules /.pnp .pnp.js +/api-go/tools/puppeteer/node_modules # testing /coverage diff --git a/README.md b/README.md index 131f793..320d11f 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,53 @@ We maintain a [list](api-go/config/multi-repo.md) which you could directly add y [![Monthly Active Contributors](https://contributor-overtime-api.apiseven.com/contributors-svg?chart=contributorMonthlyActivity&repo=apache/apisix&merge=true)](https://www.apiseven.com/en/contributor-graph?chart=contributorMonthlyActivity&repo=apache/apisix&merge=true) +## Development + +The current project uses Google Cloud deployment, to develop and debug locally, follow the steps below: + +This project depends on `Golang` and `Node.js`, please make sure you have the corresponding environment. + +Because the read and write functions of Google Clould DataStore and Google Clould DataStorage are used in the project, you need to obtain the corresponding permissions of this. + +It is recommended to download the key file of Google Clould in json format, and then set the environment variable locally: + +``` +GOOGLE_APPLICATION_CREDENTIALS=/The/path/of/json/key/file +``` + +1. Clone the project. + +``` +git clone https://github.com/api7/contributor-graph.git +``` + +2. Start the front end. + +``` +cd contributor-graph +yarn +yarn dev +``` + +3. Start API Server. + +``` +cd api-go +go run ./cmd/contributor/main.go +``` + +4. Start Google Cloud Function locally. + +This step depends on [@google-cloud/functions-framework](https://www.npmjs.com/package/@google-cloud/functions-framework) tool, please install the tool first. + +``` +cd ./api-go/tools/puppeteer +yarn +npx @google-cloud/functions-framework --target=png +``` + +Then config this address as Clould Function Trigger link in `api-go`. + ## Feature request If you have any requests, including but not limited to: diff --git a/api-go/cmd/contributor/main.go b/api-go/cmd/contributor/main.go index fdf9150..38383db 100644 --- a/api-go/cmd/contributor/main.go +++ b/api-go/cmd/contributor/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/base64" "encoding/json" "fmt" "log" @@ -130,11 +131,28 @@ func getContributorSVG(w http.ResponseWriter, r *http.Request) { return } } - w.Header().Add("content-type", "image/svg+xml;charset=utf-8") - w.Header().Add("cache-control", "public, max-age=86400") - svg = strings.Replace(svg, "%", "%%", -1) - fmt.Fprintf(w, svg) + if strings.Contains(svg, "svg") { + w.Header().Add("content-type", "image/svg+xml;charset=utf-8") + w.Header().Add("cache-control", "public, max-age=86400") + + svg = strings.Replace(svg, "%", "%%", -1) + fmt.Fprintf(w, svg) + } else { + w.Header().Add("content-type", "image/png") + w.Header().Add("cache-control", "public, max-age=86400") + + base64String := strings.Split(svg, "data:image/png;base64,")[1] + buffer, err := base64.StdEncoding.DecodeString(base64String) + + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + json.NewEncoder(w).Encode(err.Error()) + return + } + + w.Write(buffer) + } } func getRepos(w http.ResponseWriter, r *http.Request) { diff --git a/api-go/internal/graph/graph.go b/api-go/internal/graph/graph.go index c0f88a3..3f62c8e 100644 --- a/api-go/internal/graph/graph.go +++ b/api-go/internal/graph/graph.go @@ -7,8 +7,6 @@ import ( "io" "io/ioutil" "net/http" - "strconv" - "strings" "time" "cloud.google.com/go/datastore" @@ -29,7 +27,7 @@ func GenerateAndSaveSVG(ctx context.Context, repo string, merge bool, chartType } defer client.Close() - graphFunctionUrl := "https://cloudfunction.contributor-graph.com/svg?repo=" + repo + graphFunctionUrl := "https://asia-east2-api7-301102.cloudfunctions.net/png" + repo if merge { graphFunctionUrl += "&merge=true" } @@ -81,7 +79,7 @@ func GenerateAndSaveSVG(ctx context.Context, repo string, merge bool, chartType wc := client.Bucket(bucket).Object(object).NewWriter(ctx) wc.CacheControl = "public, max-age=86400" - wc.ContentType = "image/svg+xml;charset=utf-8" + wc.ContentType = "image/png" if _, err = io.Copy(wc, bytes.NewReader(svg)); err != nil { return "", fmt.Errorf("upload svg failed: io.Copy: %v", err) @@ -161,58 +159,58 @@ func SubGetSVG(w http.ResponseWriter, repo string, merge bool, charType string) // we need to also tell if the graph is ready to use on this side. // Try to get the endpoint of the line drawn and tell if it's on the right-most side func svgSucceed(svgBytes []byte) ([]byte, error) { - svg := string(svgBytes[:]) - lines := strings.Split(svg, "\n") - var svgWidth float64 - for _, l := range lines { - if strings.Contains(l, "= 0; i-- { - if strings.Contains(lines[i], renderLengthMarker) { - words := strings.Split(lines[i], " ") - svgWidthStr := fmt.Sprintf("%f", svgWidth) - for j := range words { - if words[j] == "L" && j+1 < len(words) && words[j+1] != svgWidthStr { - lines[i] = strings.ReplaceAll(lines[i], words[j+1], svgWidthStr) - break - } - } - return []byte(strings.Join(lines, "\n")), nil - } - } + // svg := string(svgBytes[:]) + // lines := strings.Split(svg, "\n") + // var svgWidth float64 + // for _, l := range lines { + // if strings.Contains(l, "= 0; i-- { + // if strings.Contains(lines[i], renderLengthMarker) { + // words := strings.Split(lines[i], " ") + // svgWidthStr := fmt.Sprintf("%f", svgWidth) + // for j := range words { + // if words[j] == "L" && j+1 < len(words) && words[j+1] != svgWidthStr { + // lines[i] = strings.ReplaceAll(lines[i], words[j+1], svgWidthStr) + // break + // } + // } + // return []byte(strings.Join(lines, "\n")), nil + // } + // } return svgBytes, nil } diff --git a/api-go/internal/utils/utils.go b/api-go/internal/utils/utils.go index 86c852b..7bf8e1c 100644 --- a/api-go/internal/utils/utils.go +++ b/api-go/internal/utils/utils.go @@ -71,7 +71,7 @@ func RepoNameToFileName(str string, merge bool, charType string) string { if charType == ContributorMonthlyActivity { filename = "monthly/" + filename } - return filename + ".svg" + return filename + ".png" } func FileNameToRepoName(str string) string { diff --git a/api-go/tools/puppeteer/README.md b/api-go/tools/puppeteer/README.md new file mode 100644 index 0000000..0c90836 --- /dev/null +++ b/api-go/tools/puppeteer/README.md @@ -0,0 +1,13 @@ +# Google Clould Function + +Google Clould Function used to generate contributor statistics picture. + +## API + +The entry point of the function is `png` in `index.js`. + +| Parameter | Required | Type | Description | Example | +| ---- | ---- | ---- | ---- | ---- | +| repo | true | string | The name of repository | apache/apisix,apache/skywalking | +| merge | false | boolean | Whether to view all repos related to this repo, when chart is `contributorMonthlyActivity`, can not be set true | true | +| chart | false | contributorOverTime contributorMonthlyActivity | chart type | | diff --git a/api-go/tools/puppeteer/fetch.js b/api-go/tools/puppeteer/fetch.js new file mode 100644 index 0000000..ede9244 --- /dev/null +++ b/api-go/tools/puppeteer/fetch.js @@ -0,0 +1,123 @@ +const moment = require('moment'); +const axios = require('axios'); + +const isSameDay = (d1, d2) => { + return ( + d1.getFullYear() === d2.getFullYear() && + d1.getMonth() === d2.getMonth() && + d1.getDate() === d2.getDate() + ); +}; + +const fetchContributorsData = (repo) => { + if (repo === "null" || repo === null) { + repo = "apache/apisix"; + } + return new Promise((resolve, reject) => { + axios.get( + `https://contributor-overtime-api.apiseven.com/contributors?repo=${repo}` + ).then(response => { + return response.data; + }).then(data => { + const { Contributors = [] } = data; + const sortContributors = Contributors.map(item => ({ + ...item, + date: item.date.substring(0, 10) + })).sort( + (a, b) => new Date(a.date).getTime() - new Date(b.date).getTime() + ); + if ( + !isSameDay( + new Date(sortContributors[sortContributors.length - 1].date), + new Date() + ) + ) { + sortContributors.push({ + repo, + idx: sortContributors[sortContributors.length - 1].idx, + date: moment(new Date()).format("YYYY-MM-DD") + }); + }; + + const processContributors = []; + sortContributors.forEach((item, index) => { + processContributors.push(item); + + if (index !== sortContributors.length - 1) { + const diffDays = moment(sortContributors[index + 1].date).diff( + item.date, + "days" + ); + if (diffDays > 1) { + for (let index = 1; index < diffDays; index++) { + processContributors.push({ + ...item, + date: moment(item.date) + .add(index, "days") + .format() + .substring(0, 10) + }); + } + } + } + }); + + const filterData = processContributors.filter( + (item, index) => + index === 0 || + index === processContributors.length - 1 || + new Date(item.date).getDate() % 10 === 5 + ); + + resolve({ repo, ...{ Contributors: filterData } }); + }).catch(error => { + reject(error); + }) + }) +}; + +const fetchMonthlyData = (repo) => { + if (repo === "null" || repo === null) { + repo = "apache/apisix"; + } + return new Promise((resolve, reject) => { + axios.get( + `https://contributor-overtime-api.apiseven.com/monthly-contributor?repo=${repo}` + ) + .then(response => { + return response.data; + }) + .then(myJson => { + resolve({ repo, ...myJson }); + }) + .catch(e => { + reject(e); + }); + }); +}; + +const fetchMergeContributor = (repo) => { + return new Promise((resolve, reject) => { + axios.get( + `https://contributor-overtime-api.apiseven.com/contributors-multi?repo=${repo.join( + "," + )}` + ) + .then(response => { + return response.data; + }) + .then(myJson => { + console.log('myJson: ', myJson); + resolve({ repo, ...myJson }); + }) + .catch(e => { + reject(e); + }); + }); +}; + +module.exports = { + fetchContributorsData, + fetchMonthlyData, + fetchMergeContributor, +} diff --git a/api-go/tools/puppeteer/index.js b/api-go/tools/puppeteer/index.js index 2159548..f65d916 100644 --- a/api-go/tools/puppeteer/index.js +++ b/api-go/tools/puppeteer/index.js @@ -1,54 +1,68 @@ +const echarts = require("echarts"); +const { createCanvas } = require('canvas'); +const { fetchContributorsData, fetchMonthlyData } = require('./fetch'); +const { updateSeries } = require('./utils'); + +const config = { + width: 896, + height: 550, +}; + /** * Responds to any HTTP request. * * @param {!express:Request} req HTTP request context. * @param {!express:Response} res HTTP response context. */ -const puppeteer = require('puppeteer'); -const querystring = require('querystring'); -exports.svg = async (req, res) => { +exports.png = (req, res) => { const repo = req.query.repo; const merge = req.query.merge; - const chart = req.query.chart; + const chartType = req.query.chart; - const browser = await puppeteer.launch({ - args: ['--no-sandbox',] + const ctx = createCanvas(config.width, config.height); + echarts.setCanvasCreator(function () { + return ctx; }); - const page = await browser.newPage(); - await page.setViewport({ width: 1920, height: 1080 }); - - graphUrl = "https://contributor-graph-git-no-animation-apiseven.vercel.app/?repo=" + repo; - if (merge) { - graphUrl += "&merge=true" - } - if (chart) { - graphUrl += "&chart=" + chart - } - - await page.goto(graphUrl); - - var getSVG = function () { - return window.echartInstance.getDataURL(); - }; - - var svgReady = function () { - return window.echartsRenderFinished; - } + let repoList = repo.split(","); + Promise.all(repoList.map(item => { + if (chartType === "contributorMonthlyActivity") { + return fetchMonthlyData(item); + }; + return fetchContributorsData(item); + })).then(data => { + const tmpDataSouce = {}; + data.forEach(item => { + const { Contributors = [], repo } = item; + const data = Contributors.map(item => { + if (chartType === "contributorMonthlyActivity") { + return { + repo, + contributorNum: item.Num, + date: item.Month + } + } else { + return { + repo, + contributorNum: item.idx, + date: item.date + } + } + }); + + if (!tmpDataSouce[item.repo]) { + tmpDataSouce[repo] = data; + } + }); - function sleep(time) { - return new Promise((resolve) => setTimeout(resolve, time)); - } + const option = updateSeries(["1970-01-01"], tmpDataSouce); + let chart = echarts.init(createCanvas(config.width, config.height)); + chart.setOption(option, true); + const base64 = chart.getDom().toDataURL(); - sleep(1000).then(() => { - var _flagCheck = setInterval(async function () { - if (await page.evaluate(svgReady) === true) { - clearInterval(_flagCheck); - var image = await page.evaluate(getSVG); - res.set('Content-Type', 'image/svg+xml'); - res.send(querystring.unescape(image).split("data:image/svg+xml;charset=UTF-8,")[1]); - await browser.close(); - } - }, 100); + res.set('Content-Type', 'image/png'); + res.send(base64); + }).catch(error => { + console.log(`generate file of ${repo} error`, error); }); }; diff --git a/api-go/tools/puppeteer/package.json b/api-go/tools/puppeteer/package.json index c9987eb..62521c2 100644 --- a/api-go/tools/puppeteer/package.json +++ b/api-go/tools/puppeteer/package.json @@ -1,7 +1,12 @@ { - "name": "sample-http", - "version": "0.0.1", - "dependencies": { - "puppeteer": "8.0.0" - } + "name": "sample-http", + "version": "0.0.1", + "dependencies": { + "@google-cloud/storage": "^5.13.1", + "axios": "^0.21.1", + "canvas": "^2.8.0", + "echarts": "5.1.2", + "lodash.clonedeep": "4.5.0", + "moment": "^2.29.1" + } } diff --git a/api-go/tools/puppeteer/utils.js b/api-go/tools/puppeteer/utils.js new file mode 100644 index 0000000..108bae9 --- /dev/null +++ b/api-go/tools/puppeteer/utils.js @@ -0,0 +1,168 @@ +const cloneDeep = require('lodash.clonedeep'); +const echarts = require("echarts"); + +const DEFAULT_COLOR = "#39a85a"; + +const generateDefaultOption = () => { + return { + color: ["#39a85a", "#4385ee", "#fabc37", "#2dc1dd", "#f972cf", "#8331c8"], + legend: { + top: "5%", + data: [], + textStyle: { + fontSize: 14 + } + }, + toolbox: { + feature: { + myShare: { + show: true, + title: "Share", + icon: "path://M830.506667 642.688c-57.344 0-104.192 30.72-126.037334 78.336l-277.162666-117.888c16.170667-25.045333 25.045333-55.722667 25.045333-88.832a167.253333 167.253333 0 0 0-4.864-40.362667l176.981333-137.301333c23.466667 17.749333 53.333333 28.245333 86.485334 28.245333 80 0 139.776-59.733333 139.776-138.88S790.912 87.04 710.954667 87.04c-80 0-139.008 59.733333-139.008 138.922667 0 20.181333 4.053333 38.741333 10.496 54.912L419.2 406.101333c-32.298667-48.469333-85.632-82.389333-146.218667-82.389333a178.944 178.944 0 0 0-179.413333 178.474667v0.853333a178.944 178.944 0 0 0 179.413333 179.2c37.12 0 71.893333-9.642667 100.181334-27.392l318.378666 135.68c4.053333 75.093333 62.250667 130.816 138.965334 130.816 80.042667 0 139.008-59.733333 139.008-138.922667 0-79.146667-59.818667-139.733333-138.965334-139.733333z", + onclick: function () { + handleShareClick(); + } + } + } + }, + dataset: [], + title: { + text: "Contributor Over Time" + }, + tooltip: { + trigger: "axis" + }, + xAxis: { + type: "time", + nameLocation: "middle", + axisLabel: { + show: true, + textStyle: { + fontSize: 14 + }, + rotate: 45 + } + }, + yAxis: { + name: "", + axisLabel: { + show: true, + textStyle: { + fontSize: 14 + } + } + }, + series: [], + grid: { + x: 10, + x2: 15, + y: 80, + containLabel: true, + y2: 5 + } + }; +}; + +const updateSeries = (passXAxis, dataSource) => { + const newClonedOption = cloneDeep( + generateDefaultOption() + ); + const datasetWithFilters = [ + ["ContributorNum", "Repo", "Date", "DateValue"] + ]; + const legend = []; + const limitDate = new Date(passXAxis[0]).getTime(); + Object.entries(dataSource).forEach(([key, value]) => { + legend.push(key); + value.forEach(item => { + datasetWithFilters.push([ + item.contributorNum, + item.repo, + item.date, + new Date(item.date).getTime() + ]); + }); + }); + + const newDateSet = datasetWithFilters.sort( + (a, b) => new Date(a[2]) - new Date(b[2]) + ); + + const filterDataset = legend.map(item => ({ + id: item, + fromDatasetId: "dataset_raw", + transform: { + type: "filter", + config: { + and: [ + { dimension: "Repo", "=": item }, + { dimension: "DateValue", gte: limitDate } + ] + } + } + })); + + const series = legend.map(item => ({ + name: item, + type: "line", + datasetId: item, + showSymbol: false, + smooth: true, + encode: { + x: "Date", + y: "ContributorNum", + itemName: "Repo", + tooltip: ["ContributorNum"] + } + })); + + if (series.length === 1) { + series[0].areaStyle = { + color: new echarts.graphic.LinearGradient(0, 0, 0, 1, [ + { + offset: 0, + color: DEFAULT_COLOR + "80" + }, + { + offset: 1, + color: DEFAULT_COLOR + "00" + } + ]) + }; + series[0].itemStyle = { + normal: { + color: DEFAULT_COLOR, + lineStyle: { + color: DEFAULT_COLOR + } + } + }; + } + + newClonedOption.dataset = [ + { + id: "dataset_raw", + source: newDateSet + } + ].concat(filterDataset); + + newClonedOption.series = series; + newClonedOption.legend.data = legend; + return newClonedOption; +}; + +const repoNameToFileName = (repo, merge, charType) => { + let fileName = repo; + if (merge) { + fileName = 'merge/' + fileName; + } + if (charType === 'contributorMonthlyActivity') { + fileName = "monthly/" + fileName; + } + return fileName + ".png"; +}; + +module.exports = { + updateSeries, + repoNameToFileName, +}