diff --git a/.editorconfig b/.editorconfig index f220a2c..cbef98b 100644 --- a/.editorconfig +++ b/.editorconfig @@ -11,3 +11,7 @@ trim_trailing_whitespace = true [*.{css}] indent_style = space indent_size = 2 + +[package{,-lock}.json] +indent_style = space +indent_size = 2 diff --git a/build/utils/compose-fetcher.js b/build/utils/compose-fetcher.js index c22725e..bd61ec1 100644 --- a/build/utils/compose-fetcher.js +++ b/build/utils/compose-fetcher.js @@ -1,5 +1,5 @@ const fs = require('fs').promises; -const fetch = require('node-fetch'); +const { graphql } = require('@octokit/graphql'); const nodeUtil = require('util'); const exec = nodeUtil.promisify(require('child_process').exec); @@ -24,6 +24,14 @@ const API_RATE_LIMIT = ` const EXEC_MAX_BUFFER = 1024 * 1024 * 32; +const graphqlWithAuth = graphql.defaults({ + headers: { + authorization: process.env.GRAPHQL_TOKEN + ? `token ${process.env.GRAPHQL_TOKEN}` + : `token ${process.env.GITHUB_TOKEN}`, + }, +}); + class DataFetcher { constructor(data_owner, data_repo) { this.data_owner = data_owner; @@ -153,46 +161,7 @@ class DataFetcher { } async fetchGithub(query, retries = 0) { - const init = {}; - init.method = "POST"; - init.headers = {}; - init.headers["Content-Type"] = "application/json"; - if (process.env.GRAPHQL_TOKEN) { - init.headers["Authorization"] = `token ${process.env.GRAPHQL_TOKEN}`; - } else if (process.env.GITHUB_TOKEN) { - init.headers["Authorization"] = `token ${process.env.GITHUB_TOKEN}`; - } - - init.body = JSON.stringify({ - query, - }); - - let res = await fetch("https://api.github.com/graphql", init); - let attempt = 0; - while (res.status !== 200 && attempt < retries) { - attempt += 1; - console.log(` Failed with status ${res.status}, retrying (${attempt}/${retries})...`); - - // GitHub API is flaky, so we add an extra delay to let it calm down a bit. - await this.delay(API_DELAY_MSEC); - res = await fetch("https://api.github.com/graphql", init); - } - - return res; - } - - async fetchGithubRest(query) { - const init = {}; - init.method = "GET"; - init.headers = {}; - init.headers["Content-Type"] = "application/json"; - if (process.env.GRAPHQL_TOKEN) { - init.headers["Authorization"] = `token ${process.env.GRAPHQL_TOKEN}`; - } else if (process.env.GITHUB_TOKEN) { - init.headers["Authorization"] = `token ${process.env.GITHUB_TOKEN}`; - } - - return await fetch(`${this.api_rest_path}${query}`, init); + return await graphqlWithAuth(query); } async checkRates() { @@ -203,18 +172,11 @@ class DataFetcher { } `; - const res = await this.fetchGithub(query); - if (res.status !== 200) { - this._handleResponseErrors(this.api_repository_id, res); - process.exitCode = buildCommon.ExitCodes.RequestFailure; - return; - } - - const data = await res.json(); + const data = await this.fetchGithub(query); await this._logResponse(data, "_rate_limit"); this._handleDataErrors(data); - const rate_limit = data.data["rateLimit"]; + const rate_limit = data["rateLimit"]; console.log(` [$${rate_limit.cost}][${rate_limit.nodeCount}] Available API calls: ${rate_limit.remaining}/${rate_limit.limit}; resets at ${rate_limit.resetAt}`); } catch (err) { console.error(" Error checking the API rate limits: " + err); @@ -309,26 +271,19 @@ class DataFetcher { console.log(` Requesting batch ${page}/${totalPages} of commit and pull request data.`); - const res = await this.fetchGithub(query, API_MAX_RETRIES); - if (res.status !== 200) { - this._handleResponseErrors(this.api_repository_id, res); - process.exitCode = buildCommon.ExitCodes.RequestFailure; - return []; - } - - const data = await res.json(); + const data = await this.fetchGithub(query); await this._logResponse(data, `data_commits`); this._handleDataErrors(data); let commit_data = {}; - for (let dataKey in data.data) { + for (let dataKey in data) { if (!dataKey.startsWith("commit_")) { continue; } - commit_data[dataKey.substring(7)] = data.data[dataKey].object; + commit_data[dataKey.substring(7)] = data[dataKey].object; } - const rate_limit = data.data["rateLimit"]; + const rate_limit = data["rateLimit"]; console.log(` [$${rate_limit.cost}][${rate_limit.nodeCount}] Retrieved ${Object.keys(commit_data).length} commits.`); console.log(` --`); return commit_data; diff --git a/package-lock.json b/package-lock.json index be31871..980266d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,10 +11,10 @@ "@babel/core": "^7.6.4", "@babel/plugin-proposal-class-properties": "^7.5.5", "@babel/plugin-proposal-decorators": "^7.6.0", + "@octokit/graphql": "^8.2.0", "dompurify": "^2.0.7", "lit-element": "^2.2.1", "marked": "^0.7.0", - "node-fetch": "^2.7.0", "posthtml": "^0.12.0", "rollup": "^1.24.0", "rollup-plugin-babel": "^4.3.3", @@ -22,7 +22,8 @@ "rollup-plugin-copy": "^3.4.0", "rollup-plugin-includepaths": "^0.2.3", "rollup-plugin-node-resolve": "^5.2.0" - } + }, + "devDependencies": {} }, "node_modules/@babel/code-frame": { "version": "7.5.5", @@ -284,6 +285,76 @@ "node": ">= 8" } }, + "node_modules/@octokit/endpoint": { + "version": "10.1.2", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-10.1.2.tgz", + "integrity": "sha512-XybpFv9Ms4hX5OCHMZqyODYqGTZ3H6K6Vva+M9LR7ib/xr1y1ZnlChYv9H680y77Vd/i/k+thXApeRASBQkzhA==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^13.6.2", + "universal-user-agent": "^7.0.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/graphql": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-8.2.0.tgz", + "integrity": "sha512-gejfDywEml/45SqbWTWrhfwvLBrcGYhOn50sPOjIeVvH6i7D16/9xcFA8dAJNp2HMcd+g4vru41g4E2RBiZvfQ==", + "license": "MIT", + "dependencies": { + "@octokit/request": "^9.1.4", + "@octokit/types": "^13.8.0", + "universal-user-agent": "^7.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/openapi-types": { + "version": "23.0.1", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-23.0.1.tgz", + "integrity": "sha512-izFjMJ1sir0jn0ldEKhZ7xegCTj/ObmEDlEfpFrx4k/JyZSMRHbO3/rBwgE7f3m2DHt+RrNGIVw4wSmwnm3t/g==", + "license": "MIT" + }, + "node_modules/@octokit/request": { + "version": "9.2.0", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-9.2.0.tgz", + "integrity": "sha512-kXLfcxhC4ozCnAXy2ff+cSxpcF0A1UqxjvYMqNuPIeOAzJbVWQ+dy5G2fTylofB/gTbObT8O6JORab+5XtA1Kw==", + "license": "MIT", + "dependencies": { + "@octokit/endpoint": "^10.0.0", + "@octokit/request-error": "^6.0.1", + "@octokit/types": "^13.6.2", + "fast-content-type-parse": "^2.0.0", + "universal-user-agent": "^7.0.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/request-error": { + "version": "6.1.6", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-6.1.6.tgz", + "integrity": "sha512-pqnVKYo/at0NuOjinrgcQYpEbv4snvP3bKMRqHaD9kIsk9u1LCpb2smHZi8/qJfgeNqLo5hNW4Z7FezNdEo0xg==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^13.6.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/types": { + "version": "13.8.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.8.0.tgz", + "integrity": "sha512-x7DjTIbEpEWXK99DMd01QfWy0hd5h4EN+Q7shkdKds3otGQP+oWE/y0A76i1OvH9fygo4ddvNf7ZvF0t78P98A==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^23.0.1" + } + }, "node_modules/@types/estree": { "version": "0.0.39", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.39.tgz", @@ -534,6 +605,22 @@ "node": ">=0.10.0" } }, + "node_modules/fast-content-type-parse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fast-content-type-parse/-/fast-content-type-parse-2.0.1.tgz", + "integrity": "sha512-nGqtvLrj5w0naR6tDPfB4cUmYCqouzyQiz6C5y/LtcDllJdrcc6WaWW6iXyIIOErTa/XRybj28aasdn4LkVk6Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, "node_modules/fast-glob": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.1.0.tgz", @@ -862,26 +949,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "license": "MIT", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -1151,11 +1218,11 @@ "node": ">=8.0" } }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "license": "MIT" + "node_modules/universal-user-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-7.0.2.tgz", + "integrity": "sha512-0JCqzSKnStlRRQfCdowvqy3cy0Dvtlb8xecj/H8JFZuCze4rwjPZQOgvFvn0Ws/usCHQFGpyr+pB9adaGwXn4Q==", + "license": "ISC" }, "node_modules/universalify": { "version": "0.1.2", @@ -1170,22 +1237,6 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "license": "BSD-2-Clause" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "license": "MIT", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", diff --git a/package.json b/package.json index 7085aa0..088d14f 100644 --- a/package.json +++ b/package.json @@ -13,10 +13,10 @@ "@babel/core": "^7.6.4", "@babel/plugin-proposal-class-properties": "^7.5.5", "@babel/plugin-proposal-decorators": "^7.6.0", + "@octokit/graphql": "^8.2.0", "dompurify": "^2.0.7", "lit-element": "^2.2.1", "marked": "^0.7.0", - "node-fetch": "^2.7.0", "posthtml": "^0.12.0", "rollup": "^1.24.0", "rollup-plugin-babel": "^4.3.3",