diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index bee39d0..0000000 --- a/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM golangci/golangci-lint:v1.26 - -COPY entrypoint.sh /entrypoint.sh - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/README.md b/README.md index 6ca3dac..07e77a7 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,14 @@ The restrictions of annotations are the following: 1. Currently, they don't support markdown formatting (see the [feature request](https://github.community/t5/GitHub-API-Development-and/Checks-Ability-to-include-Markdown-in-line-annotations/m-p/56704)) 2. They aren't shown in list of comments like it was with [golangci.com](https://golangci.com). If you would like to have comments - please, up-vote [the issue](https://github.com/golangci/golangci-lint-action/issues/5). +## Performance + +The action was implemented with performance in mind: + +1. We cache data by [@actions/cache](https://github.com/actions/cache) between builds: Go build cache, Go modules cache, golangci-lint analysis cache. +2. We don't use Docker because image pulling is slow. +3. We do as much as we can in parallel, e.g. we download cache, go and golangci-lint binary in parallel. + ## Internals We use JavaScript-based action. We don't use Docker-based action because: @@ -59,12 +67,21 @@ We use JavaScript-based action. We don't use Docker-based action because: Inside our action we perform 3 steps: 1. Setup environment running in parallel: - * restore [cache](https://github.com/actions/cache) of previous analyzes into `$HOME/.cache/golangci-lint` + * restore [cache](https://github.com/actions/cache) of previous analyzes * list [releases of golangci-lint](https://github.com/golangci/golangci-lint/releases) and find the latest patch version for needed version (users of this action can specify only minor version). After that install [golangci-lint](https://github.com/golangci/golangci-lint) using [@actions/tool-cache](https://github.com/actions/toolkit/tree/master/packages/tool-cache) * install the latest Go 1.x version using [@actions/setup-go](https://github.com/actions/setup-go) 2. Run `golangci-lint` with specified by user `args` -3. Save cache from `$HOME/.cache/golangci-lint` for later builds +3. Save cache for later builds + +### Caching internals + +1. We save and restore the following directories: `~/.cache/golangci-lint`, `~/.cache/go-build`, `~/go/pkg`. +2. The primary caching key looks like `golangci-lint.cache-{interval_number}-{go.mod_hash}`. Interval number ensures that we periodically invalidate + our cache (every 7 days). `go.mod` hash ensures that we invalidate the cache early - as soon as dependencies have changed. +3. We use [restore keys](https://help.github.com/en/actions/configuring-and-managing-workflows/caching-dependencies-to-speed-up-workflows#matching-a-cache-key): `golangci-lint.cache-{interval_number}-`, `golangci-lint.cache-`. GitHub matches keys by prefix if we have no exact match for the primary cache. + +This scheme is basic and needs improvements. Pull requests and ideas are welcome. ## Development of this action diff --git a/dist/post_run/index.js b/dist/post_run/index.js index 31a5702..20a5360 100644 --- a/dist/post_run/index.js +++ b/dist/post_run/index.js @@ -27870,29 +27870,73 @@ Object.defineProperty(exports, "__esModule", { value: true }); const core = __importStar(__webpack_require__(470)); const restore_1 = __importDefault(__webpack_require__(206)); const save_1 = __importDefault(__webpack_require__(781)); -// TODO: ensure dir exists, have access, etc -const getCacheDir = () => `${process.env.HOME}/.cache/golangci-lint`; -const setCacheInputs = () => { - process.env.INPUT_KEY = `golangci-lint.analysis-cache`; - process.env.INPUT_PATH = getCacheDir(); +const crypto = __importStar(__webpack_require__(417)); +const fs = __importStar(__webpack_require__(747)); +function checksumFile(hashName, path) { + return new Promise((resolve, reject) => { + const hash = crypto.createHash(hashName); + const stream = fs.createReadStream(path); + stream.on("error", err => reject(err)); + stream.on("data", chunk => hash.update(chunk)); + stream.on("end", () => resolve(hash.digest("hex"))); + }); +} +const pathExists = (path) => __awaiter(void 0, void 0, void 0, function* () { return !!(yield fs.promises.stat(path).catch(e => false)); }); +const getLintCacheDir = () => `${process.env.HOME}/.cache/golangci-lint`; +const getCacheDirs = () => { + // Not existing dirs are ok here: it works. + return [getLintCacheDir(), `${process.env.HOME}/.cache/go-build`, `${process.env.HOME}/go/pkg`]; }; +const getIntervalKey = (invalidationIntervalDays) => { + const now = new Date(); + const secondsSinceEpoch = now.getTime() / 1000; + const intervalNumber = Math.floor(secondsSinceEpoch / (invalidationIntervalDays * 86400)); + return intervalNumber.toString(); +}; +function buildCacheKeys() { + return __awaiter(this, void 0, void 0, function* () { + const keys = []; + let cacheKey = `golangci-lint.cache-`; + keys.push(cacheKey); + // Periodically invalidate a cache because a new code being added. + // TODO: configure it via inputs. + cacheKey += `${getIntervalKey(7)}-`; + keys.push(cacheKey); + if (yield pathExists(`go.mod`)) { + // Add checksum to key to invalidate a cache when dependencies change. + cacheKey += yield checksumFile(`cache-key`, `go.mod`); + } + else { + cacheKey += `nogomod`; + } + keys.push(cacheKey); + return keys; + }); +} function restoreCache() { return __awaiter(this, void 0, void 0, function* () { const startedAt = Date.now(); - setCacheInputs(); + const keys = yield buildCacheKeys(); + const primaryKey = keys.pop(); + const restoreKeys = keys.reverse(); + core.info(`Primary analysis cache key is '${primaryKey}', restore keys are '${restoreKeys.join(` | `)}'`); + process.env[`INPUT_RESTORE-KEYS`] = restoreKeys.join(`\n`); + process.env.INPUT_KEY = primaryKey; + process.env.INPUT_PATH = getCacheDirs().join(`\n`); // Tell golangci-lint to use our cache directory. - process.env.GOLANGCI_LINT_CACHE = getCacheDir(); + process.env.GOLANGCI_LINT_CACHE = getLintCacheDir(); yield restore_1.default(); - core.info(`Restored golangci-lint analysis cache in ${Date.now() - startedAt}ms`); + core.info(`Restored cache for golangci-lint from key '${primaryKey}' in ${Date.now() - startedAt}ms`); }); } exports.restoreCache = restoreCache; function saveCache() { return __awaiter(this, void 0, void 0, function* () { const startedAt = Date.now(); - setCacheInputs(); + const cacheDirs = getCacheDirs(); + process.env.INPUT_PATH = cacheDirs.join(`\n`); yield save_1.default(); - core.info(`Saved golangci-lint analysis cache in ${Date.now() - startedAt}ms`); + core.info(`Saved cache for golangci-lint from paths '${cacheDirs.join(`, `)}' in ${Date.now() - startedAt}ms`); }); } exports.saveCache = saveCache; diff --git a/dist/run/index.js b/dist/run/index.js index ffc8520..63ce273 100644 --- a/dist/run/index.js +++ b/dist/run/index.js @@ -27882,29 +27882,73 @@ Object.defineProperty(exports, "__esModule", { value: true }); const core = __importStar(__webpack_require__(470)); const restore_1 = __importDefault(__webpack_require__(206)); const save_1 = __importDefault(__webpack_require__(781)); -// TODO: ensure dir exists, have access, etc -const getCacheDir = () => `${process.env.HOME}/.cache/golangci-lint`; -const setCacheInputs = () => { - process.env.INPUT_KEY = `golangci-lint.analysis-cache`; - process.env.INPUT_PATH = getCacheDir(); +const crypto = __importStar(__webpack_require__(417)); +const fs = __importStar(__webpack_require__(747)); +function checksumFile(hashName, path) { + return new Promise((resolve, reject) => { + const hash = crypto.createHash(hashName); + const stream = fs.createReadStream(path); + stream.on("error", err => reject(err)); + stream.on("data", chunk => hash.update(chunk)); + stream.on("end", () => resolve(hash.digest("hex"))); + }); +} +const pathExists = (path) => __awaiter(void 0, void 0, void 0, function* () { return !!(yield fs.promises.stat(path).catch(e => false)); }); +const getLintCacheDir = () => `${process.env.HOME}/.cache/golangci-lint`; +const getCacheDirs = () => { + // Not existing dirs are ok here: it works. + return [getLintCacheDir(), `${process.env.HOME}/.cache/go-build`, `${process.env.HOME}/go/pkg`]; }; +const getIntervalKey = (invalidationIntervalDays) => { + const now = new Date(); + const secondsSinceEpoch = now.getTime() / 1000; + const intervalNumber = Math.floor(secondsSinceEpoch / (invalidationIntervalDays * 86400)); + return intervalNumber.toString(); +}; +function buildCacheKeys() { + return __awaiter(this, void 0, void 0, function* () { + const keys = []; + let cacheKey = `golangci-lint.cache-`; + keys.push(cacheKey); + // Periodically invalidate a cache because a new code being added. + // TODO: configure it via inputs. + cacheKey += `${getIntervalKey(7)}-`; + keys.push(cacheKey); + if (yield pathExists(`go.mod`)) { + // Add checksum to key to invalidate a cache when dependencies change. + cacheKey += yield checksumFile(`cache-key`, `go.mod`); + } + else { + cacheKey += `nogomod`; + } + keys.push(cacheKey); + return keys; + }); +} function restoreCache() { return __awaiter(this, void 0, void 0, function* () { const startedAt = Date.now(); - setCacheInputs(); + const keys = yield buildCacheKeys(); + const primaryKey = keys.pop(); + const restoreKeys = keys.reverse(); + core.info(`Primary analysis cache key is '${primaryKey}', restore keys are '${restoreKeys.join(` | `)}'`); + process.env[`INPUT_RESTORE-KEYS`] = restoreKeys.join(`\n`); + process.env.INPUT_KEY = primaryKey; + process.env.INPUT_PATH = getCacheDirs().join(`\n`); // Tell golangci-lint to use our cache directory. - process.env.GOLANGCI_LINT_CACHE = getCacheDir(); + process.env.GOLANGCI_LINT_CACHE = getLintCacheDir(); yield restore_1.default(); - core.info(`Restored golangci-lint analysis cache in ${Date.now() - startedAt}ms`); + core.info(`Restored cache for golangci-lint from key '${primaryKey}' in ${Date.now() - startedAt}ms`); }); } exports.restoreCache = restoreCache; function saveCache() { return __awaiter(this, void 0, void 0, function* () { const startedAt = Date.now(); - setCacheInputs(); + const cacheDirs = getCacheDirs(); + process.env.INPUT_PATH = cacheDirs.join(`\n`); yield save_1.default(); - core.info(`Saved golangci-lint analysis cache in ${Date.now() - startedAt}ms`); + core.info(`Saved cache for golangci-lint from paths '${cacheDirs.join(`, `)}' in ${Date.now() - startedAt}ms`); }); } exports.saveCache = saveCache; diff --git a/entrypoint.sh b/entrypoint.sh deleted file mode 100755 index ad37aab..0000000 --- a/entrypoint.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -echo 'golangci-lint-action: start' -echo " flags: ${INPUT_FLAGS}" -echo " format: ${INPUT_FORMAT}" - -cd "${GITHUB_WORKSPACE}/${INPUT_DIRECTORY}" || exit 1 - -# shellcheck disable=SC2086 -golangci-lint run --out-format ${INPUT_FORMAT} ${INPUT_FLAGS} diff --git a/src/cache.ts b/src/cache.ts index bfa27d2..4061c8e 100644 --- a/src/cache.ts +++ b/src/cache.ts @@ -1,29 +1,81 @@ import * as core from "@actions/core" import restore from "cache/lib/restore" import save from "cache/lib/save" +import * as crypto from "crypto" +import * as fs from "fs" -// TODO: ensure dir exists, have access, etc -const getCacheDir = (): string => `${process.env.HOME}/.cache/golangci-lint` +function checksumFile(hashName: string, path: string) { + return new Promise((resolve, reject) => { + const hash = crypto.createHash(hashName) + const stream = fs.createReadStream(path) + stream.on("error", err => reject(err)) + stream.on("data", chunk => hash.update(chunk)) + stream.on("end", () => resolve(hash.digest("hex"))) + }) +} -const setCacheInputs = (): void => { - process.env.INPUT_KEY = `golangci-lint.analysis-cache` - process.env.INPUT_PATH = getCacheDir() +const pathExists = async (path: string) => !!(await fs.promises.stat(path).catch(e => false)) + +const getLintCacheDir = (): string => `${process.env.HOME}/.cache/golangci-lint` + +const getCacheDirs = (): string[] => { + // Not existing dirs are ok here: it works. + return [getLintCacheDir(), `${process.env.HOME}/.cache/go-build`, `${process.env.HOME}/go/pkg`] +} + +const getIntervalKey = (invalidationIntervalDays: number): string => { + const now = new Date() + const secondsSinceEpoch = now.getTime() / 1000 + const intervalNumber = Math.floor(secondsSinceEpoch / (invalidationIntervalDays * 86400)) + return intervalNumber.toString() +} + +async function buildCacheKeys(): Promise { + const keys = [] + let cacheKey = `golangci-lint.cache-` + keys.push(cacheKey) + + // Periodically invalidate a cache because a new code being added. + // TODO: configure it via inputs. + cacheKey += `${getIntervalKey(7)}-` + keys.push(cacheKey) + + if (await pathExists(`go.mod`)) { + // Add checksum to key to invalidate a cache when dependencies change. + cacheKey += await checksumFile(`cache-key`, `go.mod`) + } else { + cacheKey += `nogomod` + } + keys.push(cacheKey) + + return keys } export async function restoreCache(): Promise { const startedAt = Date.now() - setCacheInputs() + + const keys = await buildCacheKeys() + const primaryKey = keys.pop() + const restoreKeys = keys.reverse() + core.info(`Primary analysis cache key is '${primaryKey}', restore keys are '${restoreKeys.join(` | `)}'`) + process.env[`INPUT_RESTORE-KEYS`] = restoreKeys.join(`\n`) + process.env.INPUT_KEY = primaryKey + + process.env.INPUT_PATH = getCacheDirs().join(`\n`) // Tell golangci-lint to use our cache directory. - process.env.GOLANGCI_LINT_CACHE = getCacheDir() + process.env.GOLANGCI_LINT_CACHE = getLintCacheDir() await restore() - core.info(`Restored golangci-lint analysis cache in ${Date.now() - startedAt}ms`) + core.info(`Restored cache for golangci-lint from key '${primaryKey}' in ${Date.now() - startedAt}ms`) } export async function saveCache(): Promise { const startedAt = Date.now() - setCacheInputs() + + const cacheDirs = getCacheDirs() + process.env.INPUT_PATH = cacheDirs.join(`\n`) + await save() - core.info(`Saved golangci-lint analysis cache in ${Date.now() - startedAt}ms`) + core.info(`Saved cache for golangci-lint from paths '${cacheDirs.join(`, `)}' in ${Date.now() - startedAt}ms`) }