actions/golangci-lint-action
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #7 from golangci/feature/improve-cache

Browse Source 
improve caching
This commit is contained in:
Isaev Denis 2020-05-05 17:06:23 +03:00 committed by GitHub
commit afc07fb790
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 189 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Dockerfile
View File

@ -1,5 +0,0 @@
FROM golangci/golangci-lint:v1.26
COPY entrypoint.sh /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]

21
README.md
View File

@ -49,6 +49,14 @@ The restrictions of annotations are the following:
1. Currently, they don't support markdown formatting (see the [feature request](https://github.community/t5/GitHub-API-Development-and/Checks-Ability-to-include-Markdown-in-line-annotations/m-p/56704))
2. They aren't shown in list of comments like it was with [golangci.com](https://golangci.com). If you would like to have comments - please, up-vote [the issue](https://github.com/golangci/golangci-lint-action/issues/5).
## Performance
The action was implemented with performance in mind:
1. We cache data by [@actions/cache](https://github.com/actions/cache) between builds: Go build cache, Go modules cache, golangci-lint analysis cache.
2. We don't use Docker because image pulling is slow.
3. We do as much as we can in parallel, e.g. we download cache, go and golangci-lint binary in parallel.
## Internals
We use JavaScript-based action. We don't use Docker-based action because:
@ -59,12 +67,21 @@ We use JavaScript-based action. We don't use Docker-based action because:
Inside our action we perform 3 steps:
1. Setup environment running in parallel:
* restore [cache](https://github.com/actions/cache) of previous analyzes into `$HOME/.cache/golangci-lint`
* restore [cache](https://github.com/actions/cache) of previous analyzes
* list [releases of golangci-lint](https://github.com/golangci/golangci-lint/releases) and find the latest patch version
for needed version (users of this action can specify only minor version). After that install [golangci-lint](https://github.com/golangci/golangci-lint) using [@actions/tool-cache](https://github.com/actions/toolkit/tree/master/packages/tool-cache)
* install the latest Go 1.x version using [@actions/setup-go](https://github.com/actions/setup-go)
2. Run `golangci-lint` with specified by user `args`
3. Save cache from `$HOME/.cache/golangci-lint` for later builds
3. Save cache for later builds
### Caching internals
1. We save and restore the following directories: `~/.cache/golangci-lint`, `~/.cache/go-build`, `~/go/pkg`.
2. The primary caching key looks like `golangci-lint.cache-{interval_number}-{go.mod_hash}`. Interval number ensures that we periodically invalidate
our cache (every 7 days). `go.mod` hash ensures that we invalidate the cache early - as soon as dependencies have changed.
3. We use [restore keys](https://help.github.com/en/actions/configuring-and-managing-workflows/caching-dependencies-to-speed-up-workflows#matching-a-cache-key): `golangci-lint.cache-{interval_number}-`, `golangci-lint.cache-`. GitHub matches keys by prefix if we have no exact match for the primary cache.
This scheme is basic and needs improvements. Pull requests and ideas are welcome.
## Development of this action

64
dist/post_run/index.js vendored
View File

@ -27870,29 +27870,73 @@ Object.defineProperty(exports, "__esModule", { value: true });
const core = __importStar(__webpack_require__(470));
const restore_1 = __importDefault(__webpack_require__(206));
const save_1 = __importDefault(__webpack_require__(781));
// TODO: ensure dir exists, have access, etc
const getCacheDir = () => `${process.env.HOME}/.cache/golangci-lint`;
const setCacheInputs = () => {
process.env.INPUT_KEY = `golangci-lint.analysis-cache`;
process.env.INPUT_PATH = getCacheDir();
const crypto = __importStar(__webpack_require__(417));
const fs = __importStar(__webpack_require__(747));
function checksumFile(hashName, path) {
return new Promise((resolve, reject) => {
const hash = crypto.createHash(hashName);
const stream = fs.createReadStream(path);
stream.on("error", err => reject(err));
stream.on("data", chunk => hash.update(chunk));
stream.on("end", () => resolve(hash.digest("hex")));
});
}
const pathExists = (path) => __awaiter(void 0, void 0, void 0, function* () { return !!(yield fs.promises.stat(path).catch(e => false)); });
const getLintCacheDir = () => `${process.env.HOME}/.cache/golangci-lint`;
const getCacheDirs = () => {
// Not existing dirs are ok here: it works.
return [getLintCacheDir(), `${process.env.HOME}/.cache/go-build`, `${process.env.HOME}/go/pkg`];
};
const getIntervalKey = (invalidationIntervalDays) => {
const now = new Date();
const secondsSinceEpoch = now.getTime() / 1000;
const intervalNumber = Math.floor(secondsSinceEpoch / (invalidationIntervalDays * 86400));
return intervalNumber.toString();
};
function buildCacheKeys() {
return __awaiter(this, void 0, void 0, function* () {
const keys = [];
let cacheKey = `golangci-lint.cache-`;
keys.push(cacheKey);
// Periodically invalidate a cache because a new code being added.
// TODO: configure it via inputs.
cacheKey += `${getIntervalKey(7)}-`;
keys.push(cacheKey);
if (yield pathExists(`go.mod`)) {
// Add checksum to key to invalidate a cache when dependencies change.
cacheKey += yield checksumFile(`cache-key`, `go.mod`);
}
else {
cacheKey += `nogomod`;
}
keys.push(cacheKey);
return keys;
});
}
function restoreCache() {
return __awaiter(this, void 0, void 0, function* () {
const startedAt = Date.now();
setCacheInputs();
const keys = yield buildCacheKeys();
const primaryKey = keys.pop();
const restoreKeys = keys.reverse();
core.info(`Primary analysis cache key is '${primaryKey}', restore keys are '${restoreKeys.join(` | `)}'`);
process.env[`INPUT_RESTORE-KEYS`] = restoreKeys.join(`\n`);
process.env.INPUT_KEY = primaryKey;
process.env.INPUT_PATH = getCacheDirs().join(`\n`);
// Tell golangci-lint to use our cache directory.
process.env.GOLANGCI_LINT_CACHE = getCacheDir();
process.env.GOLANGCI_LINT_CACHE = getLintCacheDir();
yield restore_1.default();
core.info(`Restored golangci-lint analysis cache in ${Date.now() - startedAt}ms`);
core.info(`Restored cache for golangci-lint from key '${primaryKey}' in ${Date.now() - startedAt}ms`);
});
}
exports.restoreCache = restoreCache;
function saveCache() {
return __awaiter(this, void 0, void 0, function* () {
const startedAt = Date.now();
setCacheInputs();
const cacheDirs = getCacheDirs();
process.env.INPUT_PATH = cacheDirs.join(`\n`);
yield save_1.default();
core.info(`Saved golangci-lint analysis cache in ${Date.now() - startedAt}ms`);
core.info(`Saved cache for golangci-lint from paths '${cacheDirs.join(`, `)}' in ${Date.now() - startedAt}ms`);
});
}
exports.saveCache = saveCache;

64
dist/run/index.js vendored
View File

@ -27882,29 +27882,73 @@ Object.defineProperty(exports, "__esModule", { value: true });
const core = __importStar(__webpack_require__(470));
const restore_1 = __importDefault(__webpack_require__(206));
const save_1 = __importDefault(__webpack_require__(781));
// TODO: ensure dir exists, have access, etc
const getCacheDir = () => `${process.env.HOME}/.cache/golangci-lint`;
const setCacheInputs = () => {
process.env.INPUT_KEY = `golangci-lint.analysis-cache`;
process.env.INPUT_PATH = getCacheDir();
const crypto = __importStar(__webpack_require__(417));
const fs = __importStar(__webpack_require__(747));
function checksumFile(hashName, path) {
return new Promise((resolve, reject) => {
const hash = crypto.createHash(hashName);
const stream = fs.createReadStream(path);
stream.on("error", err => reject(err));
stream.on("data", chunk => hash.update(chunk));
stream.on("end", () => resolve(hash.digest("hex")));
});
}
const pathExists = (path) => __awaiter(void 0, void 0, void 0, function* () { return !!(yield fs.promises.stat(path).catch(e => false)); });
const getLintCacheDir = () => `${process.env.HOME}/.cache/golangci-lint`;
const getCacheDirs = () => {
// Not existing dirs are ok here: it works.
return [getLintCacheDir(), `${process.env.HOME}/.cache/go-build`, `${process.env.HOME}/go/pkg`];
};
const getIntervalKey = (invalidationIntervalDays) => {
const now = new Date();
const secondsSinceEpoch = now.getTime() / 1000;
const intervalNumber = Math.floor(secondsSinceEpoch / (invalidationIntervalDays * 86400));
return intervalNumber.toString();
};
function buildCacheKeys() {
return __awaiter(this, void 0, void 0, function* () {
const keys = [];
let cacheKey = `golangci-lint.cache-`;
keys.push(cacheKey);
// Periodically invalidate a cache because a new code being added.
// TODO: configure it via inputs.
cacheKey += `${getIntervalKey(7)}-`;
keys.push(cacheKey);
if (yield pathExists(`go.mod`)) {
// Add checksum to key to invalidate a cache when dependencies change.
cacheKey += yield checksumFile(`cache-key`, `go.mod`);
}
else {
cacheKey += `nogomod`;
}
keys.push(cacheKey);
return keys;
});
}
function restoreCache() {
return __awaiter(this, void 0, void 0, function* () {
const startedAt = Date.now();
setCacheInputs();
const keys = yield buildCacheKeys();
const primaryKey = keys.pop();
const restoreKeys = keys.reverse();
core.info(`Primary analysis cache key is '${primaryKey}', restore keys are '${restoreKeys.join(` | `)}'`);
process.env[`INPUT_RESTORE-KEYS`] = restoreKeys.join(`\n`);
process.env.INPUT_KEY = primaryKey;
process.env.INPUT_PATH = getCacheDirs().join(`\n`);
// Tell golangci-lint to use our cache directory.
process.env.GOLANGCI_LINT_CACHE = getCacheDir();
process.env.GOLANGCI_LINT_CACHE = getLintCacheDir();
yield restore_1.default();
core.info(`Restored golangci-lint analysis cache in ${Date.now() - startedAt}ms`);
core.info(`Restored cache for golangci-lint from key '${primaryKey}' in ${Date.now() - startedAt}ms`);
});
}
exports.restoreCache = restoreCache;
function saveCache() {
return __awaiter(this, void 0, void 0, function* () {
const startedAt = Date.now();
setCacheInputs();
const cacheDirs = getCacheDirs();
process.env.INPUT_PATH = cacheDirs.join(`\n`);
yield save_1.default();
core.info(`Saved golangci-lint analysis cache in ${Date.now() - startedAt}ms`);
core.info(`Saved cache for golangci-lint from paths '${cacheDirs.join(`, `)}' in ${Date.now() - startedAt}ms`);
});
}
exports.saveCache = saveCache;

10
entrypoint.sh
View File

@ -1,10 +0,0 @@
#!/bin/bash
echo 'golangci-lint-action: start'
echo " flags: ${INPUT_FLAGS}"
echo " format: ${INPUT_FORMAT}"
cd "${GITHUB_WORKSPACE}/${INPUT_DIRECTORY}" || exit 1
# shellcheck disable=SC2086
golangci-lint run --out-format ${INPUT_FORMAT} ${INPUT_FLAGS}

72
src/cache.ts
View File

@ -1,29 +1,81 @@
import * as core from "@actions/core"
import restore from "cache/lib/restore"
import save from "cache/lib/save"
import * as crypto from "crypto"
import * as fs from "fs"
// TODO: ensure dir exists, have access, etc
const getCacheDir = (): string => `${process.env.HOME}/.cache/golangci-lint`
function checksumFile(hashName: string, path: string) {
return new Promise((resolve, reject) => {
const hash = crypto.createHash(hashName)
const stream = fs.createReadStream(path)
stream.on("error", err => reject(err))
stream.on("data", chunk => hash.update(chunk))
stream.on("end", () => resolve(hash.digest("hex")))
})
}
const setCacheInputs = (): void => {
process.env.INPUT_KEY = `golangci-lint.analysis-cache`
process.env.INPUT_PATH = getCacheDir()
const pathExists = async (path: string) => !!(await fs.promises.stat(path).catch(e => false))
const getLintCacheDir = (): string => `${process.env.HOME}/.cache/golangci-lint`
const getCacheDirs = (): string[] => {
// Not existing dirs are ok here: it works.
return [getLintCacheDir(), `${process.env.HOME}/.cache/go-build`, `${process.env.HOME}/go/pkg`]
}
const getIntervalKey = (invalidationIntervalDays: number): string => {
const now = new Date()
const secondsSinceEpoch = now.getTime() / 1000
const intervalNumber = Math.floor(secondsSinceEpoch / (invalidationIntervalDays * 86400))
return intervalNumber.toString()
}
async function buildCacheKeys(): Promise<string[]> {
const keys = []
let cacheKey = `golangci-lint.cache-`
keys.push(cacheKey)
// Periodically invalidate a cache because a new code being added.
// TODO: configure it via inputs.
cacheKey += `${getIntervalKey(7)}-`
keys.push(cacheKey)
if (await pathExists(`go.mod`)) {
// Add checksum to key to invalidate a cache when dependencies change.
cacheKey += await checksumFile(`cache-key`, `go.mod`)
} else {
cacheKey += `nogomod`
}
keys.push(cacheKey)
return keys
}
export async function restoreCache(): Promise<void> {
const startedAt = Date.now()
setCacheInputs()
const keys = await buildCacheKeys()
const primaryKey = keys.pop()
const restoreKeys = keys.reverse()
core.info(`Primary analysis cache key is '${primaryKey}', restore keys are '${restoreKeys.join(` | `)}'`)
process.env[`INPUT_RESTORE-KEYS`] = restoreKeys.join(`\n`)
process.env.INPUT_KEY = primaryKey
process.env.INPUT_PATH = getCacheDirs().join(`\n`)
// Tell golangci-lint to use our cache directory.
process.env.GOLANGCI_LINT_CACHE = getCacheDir()
process.env.GOLANGCI_LINT_CACHE = getLintCacheDir()
await restore()
core.info(`Restored golangci-lint analysis cache in ${Date.now() - startedAt}ms`)
core.info(`Restored cache for golangci-lint from key '${primaryKey}' in ${Date.now() - startedAt}ms`)
}
export async function saveCache(): Promise<void> {
const startedAt = Date.now()
setCacheInputs()
const cacheDirs = getCacheDirs()
process.env.INPUT_PATH = cacheDirs.join(`\n`)
await save()
core.info(`Saved golangci-lint analysis cache in ${Date.now() - startedAt}ms`)
core.info(`Saved cache for golangci-lint from paths '${cacheDirs.join(`, `)}' in ${Date.now() - startedAt}ms`)
}