chore(release): v1.0.40

fix: fixes while exit condition
feat: paginates using async generator
2021-10-13 22:22:11 +00:00 · 2021-10-14 00:20:38 +02:00 · 2021-10-13 23:44:05 +02:00 · 2021-08-31 09:17:17 +00:00 · 2021-08-31 11:16:32 +02:00 · 2021-08-12 23:57:13 +00:00
8 changed files with 223 additions and 171 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,35 +1,36 @@
-## <small>1.0.35 (2021-06-22)</small>
+## <small>1.0.40 (2021-10-13)</small>

-* build(deps): bump set-getter from 0.1.0 to 0.1.1 ([970d603](https://github.com/simonecorsi/mawesome/commit/970d603))
+* fix: fixes while exit condition ([8f9b4cc](https://github.com/simonecorsi/mawesome/commit/8f9b4cc))
+* feat: paginates using async generator ([51da6c2](https://github.com/simonecorsi/mawesome/commit/51da6c2))



-## <small>1.0.34 (2021-05-19)</small>
+## <small>1.0.39 (2021-08-31)</small>

-* chore(release): v1.0.34 ([ab02c0d](https://github.com/simonecorsi/mawesome/commit/ab02c0d))
+* chore(release): v1.0.39 ([f7a8341](https://github.com/simonecorsi/mawesome/commit/f7a8341))
+* build(deps): bump path-parse from 1.0.6 to 1.0.7 ([ef137b5](https://github.com/simonecorsi/mawesome/commit/ef137b5))



-## <small>1.0.33 (2021-05-19)</small>
+## <small>1.0.38 (2021-06-23)</small>

-* chore(release): v1.0.33 ([fc187df](https://github.com/simonecorsi/mawesome/commit/fc187df))
-* build(deps): bump handlebars from 4.7.6 to 4.7.7 ([012928b](https://github.com/simonecorsi/mawesome/commit/012928b))
-* build(deps): bump lodash from 4.17.20 to 4.17.21 ([1015179](https://github.com/simonecorsi/mawesome/commit/1015179))
+* chore(release): v1.0.38 ([3e66d85](https://github.com/simonecorsi/mawesome/commit/3e66d85))



-## <small>1.0.32 (2021-01-22)</small>
+## <small>1.0.37 (2021-06-23)</small>

-* chore: removed unused npm packages ([3ad1555](https://github.com/simonecorsi/mawesome/commit/3ad1555))
-* chore(release): v1.0.32 ([fe15d93](https://github.com/simonecorsi/mawesome/commit/fe15d93))
-* docs: added documentation ([5753aef](https://github.com/simonecorsi/mawesome/commit/5753aef))
+* chore(release): v1.0.37 ([1643092](https://github.com/simonecorsi/mawesome/commit/1643092))
+* build(deps): bump glob-parent from 5.1.1 to 5.1.2 ([e558e07](https://github.com/simonecorsi/mawesome/commit/e558e07))
+* build(deps): bump hosted-git-info from 2.8.8 to 2.8.9 ([110a667](https://github.com/simonecorsi/mawesome/commit/110a667))
+* build(deps): bump normalize-url from 4.5.0 to 4.5.1 ([202d6fe](https://github.com/simonecorsi/mawesome/commit/202d6fe))



-## <small>1.0.31 (2021-01-21)</small>
+## <small>1.0.36 (2021-06-23)</small>

-* chore(release): v1.0.31 ([903c070](https://github.com/simonecorsi/mawesome/commit/903c070))
-* feat: added user/email input for commiting ([58a7dc1](https://github.com/simonecorsi/mawesome/commit/58a7dc1))
+* chore(release): v1.0.36 ([9c0b622](https://github.com/simonecorsi/mawesome/commit/9c0b622))
+* feat: saving json data for future use ([7ceb9cb](https://github.com/simonecorsi/mawesome/commit/7ceb9cb))



--- a/index.js
+++ b/index.js
@@ -18482,7 +18482,7 @@ const testParameter = (name, filters) => {
 };

 const normalizeDataURL = (urlString, {stripHash}) => {
-	const parts = urlString.match(/^data:(.*?),(.*?)(?:#(.*))?$/);
+	const parts = urlString.match(/^data:([^,]*?),([^#]*?)(?:#(.*))?$/);

 	if (!parts) {
 		throw new Error(`Invalid URL: ${urlString}`);
@@ -21356,12 +21356,32 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
 };
+var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); }
+var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var g = generator.apply(thisArg, _arguments || []), i, q = [];
+    return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i;
+    function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; }
+    function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
+    function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }
+    function fulfill(value) { resume("next", value); }
+    function reject(value) { resume("throw", value); }
+    function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 var _a;
 Object.defineProperty(exports, "__esModule", ({ value: true }));
-exports.pushNewFile = exports.OUTPUT_FILENAME = exports.generateMd = exports.paginate = exports.API_STARRED_URL = exports.REPO_USERNAME = exports.apiGetStar = exports.renderer = exports.isLastPage = exports.wait = void 0;
+exports.pushNewFiles = exports.MARKDOWN_FILENAME = exports.generateMd = exports.apiGetStar = exports.getNextPage = exports.renderer = exports.wait = exports.API_STARRED_URL = exports.REPO_USERNAME = void 0;
+const fs_1 = __importDefault(__nccwpck_require__(5747));
 const ejs_1 = __importDefault(__nccwpck_require__(8431));
 const core = __importStar(__nccwpck_require__(2186));
 const remark_1 = __importDefault(__nccwpck_require__(2081));
@@ -21370,16 +21390,13 @@ const template_1 = __importDefault(__nccwpck_require__(3932));
 const api_1 = __importDefault(__nccwpck_require__(8229));
 const link_1 = __importDefault(__nccwpck_require__(9338));
 const git_1 = __importDefault(__nccwpck_require__(6350));
-const fs_1 = __importDefault(__nccwpck_require__(5747));
+exports.REPO_USERNAME = (_a = process.env.GITHUB_REPOSITORY) === null || _a === void 0 ? void 0 : _a.split('/')[0];
+exports.API_STARRED_URL = `${process.env.GITHUB_API_URL}/users/${exports.REPO_USERNAME}/starred`;
 const fsp = fs_1.default.promises;
 function wait(time = 200) {
    return new Promise((resolve) => setTimeout(resolve, time));
 }
 exports.wait = wait;
-function isLastPage(links) {
-    return links.next === links.last;
-}
-exports.isLastPage = isLastPage;
 function renderer(data, templateString = template_1.default) {
    return __awaiter(this, void 0, void 0, function* () {
        try {
@@ -21392,32 +21409,62 @@ function renderer(data, templateString = template_1.default) {
    });
 }
 exports.renderer = renderer;
-function apiGetStar(url) {
+function getNextPage(links) {
+    const next = links.find((l) => l.rel === 'next');
+    const last = links.find((l) => l.rel === 'last');
+    if (!next || !last)
+        return null;
+    const matchNext = next.uri.match(/page=([0-9]*)/);
+    const matchLast = next.uri.match(/page=([0-9]*)/);
+    if (!matchNext || !matchLast)
+        return null;
+    if (matchNext[1] === matchLast[1])
+        return null;
+    return matchNext[1];
+}
+exports.getNextPage = getNextPage;
+function paginateStars(url) {
+    return __asyncGenerator(this, arguments, function* paginateStars_1() {
+        let nextPage = '1';
+        while (nextPage) {
+            try {
+                const { headers, body } = yield __await(api_1.default.get(url, {
+                    searchParams: {
+                        page: nextPage,
+                    },
+                }));
+                yield yield __await(body);
+                nextPage = getNextPage(link_1.default.parse(headers.link).refs);
+                yield __await(wait(1000)); // avoid limits
+            }
+            catch (e) {
+                console.error(e);
+                break;
+            }
+        }
+    });
+}
+function apiGetStar(url = exports.API_STARRED_URL) {
+    var e_1, _a;
    return __awaiter(this, void 0, void 0, function* () {
-        const { headers, body } = yield api_1.default.get(url);
-        return {
-            data: body,
-            links: link_1.default.parse(headers.link).refs.reduce((acc, val) => (Object.assign(Object.assign({}, acc), { [val.rel]: val.uri })), {}),
-        };
+        let data = [];
+        try {
+            for (var _b = __asyncValues(paginateStars(url)), _c; _c = yield _b.next(), !_c.done;) {
+                const stars = _c.value;
+                data = data.concat(stars);
+            }
+        }
+        catch (e_1_1) { e_1 = { error: e_1_1 }; }
+        finally {
+            try {
+                if (_c && !_c.done && (_a = _b.return)) yield _a.call(_b);
+            }
+            finally { if (e_1) throw e_1.error; }
+        }
+        return data;
    });
 }
 exports.apiGetStar = apiGetStar;
-exports.REPO_USERNAME = (_a = process.env.GITHUB_REPOSITORY) === null || _a === void 0 ? void 0 : _a.split('/')[0];
-exports.API_STARRED_URL = `${process.env.GITHUB_API_URL}/users/${exports.REPO_USERNAME}/starred`;
-let links = {
-    next: exports.API_STARRED_URL,
-    last: undefined,
-};
-function paginate() {
-    return __awaiter(this, void 0, void 0, function* () {
-        if (isLastPage(links))
-            return null;
-        const r = yield apiGetStar(links.next);
-        links = r.links;
-        return r;
-    });
-}
-exports.paginate = paginate;
 function generateMd(data) {
    return new Promise((resolve) => {
        remark_1.default()
@@ -21433,17 +21480,21 @@ function generateMd(data) {
    });
 }
 exports.generateMd = generateMd;
-exports.OUTPUT_FILENAME = core.getInput('output-filename') || 'README.md';
-function pushNewFile(markdown) {
+exports.MARKDOWN_FILENAME = core.getInput('output-filename') || 'README.md';
+function pushNewFiles(files = []) {
    return __awaiter(this, void 0, void 0, function* () {
-        yield fsp.writeFile(exports.OUTPUT_FILENAME, markdown);
+        if (!files.length)
+            return;
        yield git_1.default.pull();
-        yield git_1.default.add(exports.OUTPUT_FILENAME);
-        yield git_1.default.commit(`chore(${exports.OUTPUT_FILENAME}): updated ${exports.OUTPUT_FILENAME}`);
+        yield Promise.all(files.map(({ filename, data }) => __awaiter(this, void 0, void 0, function* () {
+            yield fsp.writeFile(filename, data);
+            yield git_1.default.add(filename);
+            yield git_1.default.commit(`chore(${filename}): updated ${filename}`);
+        })));
        yield git_1.default.push();
    });
 }
-exports.pushNewFile = pushNewFile;
+exports.pushNewFiles = pushNewFiles;


 /***/ }),
@@ -21487,14 +21538,7 @@ const core = __importStar(__nccwpck_require__(2186));
 const helpers_1 = __nccwpck_require__(3015);
 function main() {
    return __awaiter(this, void 0, void 0, function* () {
-        let results = [];
-        while (true) {
-            // sorry.
-            const r = yield helpers_1.paginate();
-            if (!r || r === null)
-                break;
-            results = results.concat(r.data);
-        }
+        const results = yield helpers_1.apiGetStar();
        const sortedByLanguages = results.reduce((acc, val) => {
            const language = val.language || 'generic';
            if (!acc[language]) {
@@ -21511,7 +21555,16 @@ function main() {
            updatedAt: Date.now(),
        });
        const markdown = yield helpers_1.generateMd(rendered);
-        yield helpers_1.pushNewFile(markdown);
+        yield helpers_1.pushNewFiles([
+            {
+                filename: helpers_1.MARKDOWN_FILENAME,
+                data: markdown,
+            },
+            {
+                filename: 'data.json',
+                data: JSON.stringify(sortedByLanguages, null, 2),
+            },
+        ]);
    });
 }
 exports.main = main;
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,6 +1,6 @@
 {
  "name": "mawesome",
-  "version": "1.0.34",
+  "version": "1.0.38",
  "lockfileVersion": 1,
  "requires": true,
  "dependencies": {
@@ -2339,9 +2339,9 @@
          }
        },
        "hosted-git-info": {
-          "version": "2.8.8",
-          "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.8.tgz",
-          "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==",
+          "version": "2.8.9",
+          "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz",
+          "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==",
          "dev": true
        },
        "indent-string": {
@@ -3980,9 +3980,9 @@
          "dev": true
        },
        "hosted-git-info": {
-          "version": "2.8.8",
-          "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.8.tgz",
-          "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==",
+          "version": "2.8.9",
+          "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz",
+          "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==",
          "dev": true
        },
        "indent-string": {
@@ -4263,9 +4263,9 @@
      }
    },
    "glob-parent": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz",
-      "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==",
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
+      "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
      "dev": true,
      "requires": {
        "is-glob": "^4.0.1"
@@ -4532,9 +4532,9 @@
      }
    },
    "hosted-git-info": {
-      "version": "3.0.7",
-      "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-3.0.7.tgz",
-      "integrity": "sha512-fWqc0IcuXs+BmE9orLDyVykAG9GJtGLGuZAAqgcckPgv5xad4AcXGIv8galtQvlwutxSlaMcdw7BUtq2EIvqCQ==",
+      "version": "3.0.8",
+      "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-3.0.8.tgz",
+      "integrity": "sha512-aXpmwoOhRBrw6X3j0h5RloK4x1OzsxMPyxqIHyNfSe2pypkVTZFpEiRoSipPEPlMrh0HW/XsjkJ5WgnCirpNUw==",
      "dev": true,
      "requires": {
        "lru-cache": "^6.0.0"
@@ -6180,9 +6180,9 @@
      "dev": true
    },
    "normalize-url": {
-      "version": "4.5.0",
-      "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz",
-      "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ=="
+      "version": "4.5.1",
+      "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.1.tgz",
+      "integrity": "sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA=="
    },
    "npm-run-path": {
      "version": "4.0.1",
@@ -6668,9 +6668,9 @@
      "dev": true
    },
    "path-parse": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz",
-      "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==",
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz",
+      "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
      "dev": true
    },
    "path-to-regexp": {
@@ -6944,9 +6944,9 @@
      },
      "dependencies": {
        "hosted-git-info": {
-          "version": "2.8.8",
-          "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.8.tgz",
-          "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==",
+          "version": "2.8.9",
+          "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz",
+          "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==",
          "dev": true
        },
        "normalize-package-data": {
@@ -7922,9 +7922,9 @@
          "dev": true
        },
        "hosted-git-info": {
-          "version": "2.8.8",
-          "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.8.tgz",
-          "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==",
+          "version": "2.8.9",
+          "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz",
+          "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==",
          "dev": true
        },
        "load-json-file": {
@@ -8490,9 +8490,9 @@
          "dev": true
        },
        "hosted-git-info": {
-          "version": "2.8.8",
-          "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.8.tgz",
-          "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==",
+          "version": "2.8.9",
+          "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz",
+          "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==",
          "dev": true
        },
        "indent-string": {
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
  "name": "mawesome",
-  "version": "1.0.35",
+  "version": "1.0.40",
  "description": "Generate awesome list from user starred repositories",
  "main": "index.js",
  "author": "Simone Corsi<simonecorsi.dev@gmail.com>",
--- a/src/helpers.ts
+++ b/src/helpers.ts
@@ -1,3 +1,4 @@
+import fs from 'fs';
 import ejs from 'ejs';
 import * as core from '@actions/core';
 import remark from 'remark';
@@ -8,9 +9,10 @@ import GithubApi from './api';
 import link from './link';
 import git from './git';

-import type { PaginationLink, ApiGetStarResponse } from './types';
+import type { PaginationLink, ApiGetStarResponse, Stars } from './types';

-import fs from 'fs';
+export const REPO_USERNAME = process.env.GITHUB_REPOSITORY?.split('/')[0];
+export const API_STARRED_URL = `${process.env.GITHUB_API_URL}/users/${REPO_USERNAME}/starred`;

 const fsp = fs.promises;

@@ -18,10 +20,6 @@ export function wait(time = 200): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, time));
 }

-export function isLastPage(links: PaginationLink): boolean {
-  return links.next === links.last;
-}
-
 export async function renderer(
  data: { [key: string]: any },
  templateString = MD_TEMPLATE
@@ -34,32 +32,45 @@ export async function renderer(
  }
 }

-export async function apiGetStar(url: string): Promise<ApiGetStarResponse> {
-  const { headers, body }: any = await GithubApi.get(url);
-  return {
-    data: body,
-    links: link.parse(headers.link).refs.reduce(
-      (acc, val) => ({
-        ...acc,
-        [val.rel]: val.uri,
-      }),
-      {}
-    ),
-  };
+export function getNextPage(links: PaginationLink[]): string | null {
+  const next = links.find((l) => l.rel === 'next');
+  const last = links.find((l) => l.rel === 'last');
+  if (!next || !last) return null;
+  const matchNext = next.uri.match(/page=([0-9]*)/);
+  const matchLast = next.uri.match(/page=([0-9]*)/);
+
+  if (!matchNext || !matchLast) return null;
+  if (matchNext[1] === matchLast[1]) return null;
+  return matchNext[1];
 }

-export const REPO_USERNAME = process.env.GITHUB_REPOSITORY?.split('/')[0];
-export const API_STARRED_URL = `${process.env.GITHUB_API_URL}/users/${REPO_USERNAME}/starred`;
+async function* paginateStars(url: string): AsyncGenerator<Stars> {
+  let nextPage: string | null = '1';
+  while (nextPage) {
+    try {
+      const { headers, body } = await GithubApi.get(url, {
+        searchParams: {
+          page: nextPage,
+        },
+      });
+      yield (body as unknown) as Stars;
+      nextPage = getNextPage(link.parse(headers.link).refs);
+      await wait(1000); // avoid limits
+    } catch (e) {
+      console.error(e);
+      break;
+    }
+  }
+}

-let links: PaginationLink = {
-  next: API_STARRED_URL,
-  last: undefined,
-};
-export async function paginate(): Promise<ApiGetStarResponse | null> {
-  if (isLastPage(links)) return null;
-  const r = await apiGetStar(links.next);
-  links = r.links;
-  return r;
+export async function apiGetStar(
+  url: string = API_STARRED_URL
+): Promise<ApiGetStarResponse> {
+  let data: Stars[] = [];
+  for await (const stars of paginateStars(url)) {
+    data = data.concat(stars);
+  }
+  return (data as unknown) as Stars;
 }

 export function generateMd(data: string): Promise<string> {
@@ -77,12 +88,26 @@ export function generateMd(data: string): Promise<string> {
  });
 }

-export const OUTPUT_FILENAME: string =
+export const MARKDOWN_FILENAME: string =
  core.getInput('output-filename') || 'README.md';
-export async function pushNewFile(markdown: string): Promise<any> {
-  await fsp.writeFile(OUTPUT_FILENAME, markdown);
+
+type File = {
+  filename: string;
+  data: string;
+};
+
+export async function pushNewFiles(files: File[] = []): Promise<any> {
+  if (!files.length) return;
+
  await git.pull();
-  await git.add(OUTPUT_FILENAME);
-  await git.commit(`chore(${OUTPUT_FILENAME}): updated ${OUTPUT_FILENAME}`);
+
+  await Promise.all(
+    files.map(async ({ filename, data }) => {
+      await fsp.writeFile(filename, data);
+      await git.add(filename);
+      await git.commit(`chore(${filename}): updated ${filename}`);
+    })
+  );
+
  await git.push();
 }
--- a/src/index.ts
+++ b/src/index.ts
@@ -2,23 +2,17 @@ import * as core from '@actions/core';

 import {
  renderer,
-  paginate,
  REPO_USERNAME,
  generateMd,
-  pushNewFile,
+  pushNewFiles,
+  MARKDOWN_FILENAME,
+  apiGetStar,
 } from './helpers';

 import type { SortedLanguageList, Stars, Star } from './types';

 export async function main(): Promise<any> {
-  let results: Stars = [];
-
-  while (true) {
-    // sorry.
-    const r = await paginate();
-    if (!r || r === null) break;
-    results = results.concat(r.data);
-  }
+  const results: Stars = await apiGetStar();

  const sortedByLanguages = results.reduce(
    (acc: SortedLanguageList, val: Star) => {
@@ -41,7 +35,16 @@ export async function main(): Promise<any> {

  const markdown: string = await generateMd(rendered);

-  await pushNewFile(markdown);
+  await pushNewFiles([
+    {
+      filename: MARKDOWN_FILENAME,
+      data: markdown,
+    },
+    {
+      filename: 'data.json',
+      data: JSON.stringify(sortedByLanguages, null, 2),
+    },
+  ]);
 }

 export async function run(): Promise<any> {
--- a/src/types.ts
+++ b/src/types.ts
@@ -5,14 +5,11 @@ export type SortedLanguageList = {
 };

 export type PaginationLink = {
-  next: string;
-  last: string | undefined | null;
+  uri: string;
+  rel: 'next' | 'last' | 'prev' | 'first';
 };

 export type Stars = Endpoints['GET /user/starred']['response']['data'];
 export type Star = Stars[number] | { language: string };

-export type ApiGetStarResponse = {
-  links: PaginationLink;
-  data: Stars;
-};
+export type ApiGetStarResponse = Stars;
--- a/test/helpers.spec.ts
+++ b/test/helpers.spec.ts
@@ -7,7 +7,7 @@ sinon.replace(core, 'getInput', sinon.fake());

 import GithubApi from '../src/api';
 const GithubApiFake = sinon.fake((rul) => ({
-  body: 'data',
+  body: [],
  headers: {
    link:
      '<https://api.github.com/user/5617452/starred?page=2>; rel="next", <https://api.github.com/user/5617452/starred?page=2>; rel="last"',
@@ -32,13 +32,11 @@ const writeFile = sinon.fake();
 sinon.replace(fsp, 'writeFile', writeFile);

 import {
-  isLastPage,
  wait,
  renderer,
  apiGetStar,
-  paginate,
  generateMd,
-  pushNewFile,
+  pushNewFiles,
 } from '../src/helpers';

 test('wait should wait', async (t) => {
@@ -46,40 +44,15 @@ test('wait should wait', async (t) => {
  t.pass();
 });

-test('isLastPage', (t) => {
-  t.true(
-    isLastPage({
-      next: 'last',
-      last: 'last',
-    })
-  );
-  t.false(
-    isLastPage({
-      next: 'last',
-      last: undefined,
-    })
-  );
-});
-
 test('renderer should render', async (t) => {
  const output = await renderer({ variable: 123 }, 'Test: <%= variable %>');
  t.is(output, 'Test: 123');
 });

 test('apiGetStar', async (t) => {
-  let { data, links } = await apiGetStar('url');
+  let stars = await apiGetStar('url');
  t.true(GithubApiFake.called);
-  t.is(data, 'data');
-  t.deepEqual(links, {
-    last: 'https://api.github.com/user/5617452/starred?page=2',
-    next: 'https://api.github.com/user/5617452/starred?page=2',
-  });
-});
-
-test('paginate', async (t) => {
-  await paginate();
-  const res = await paginate();
-  t.is(res, null);
+  t.true(Array.isArray(stars));
 });

 test('generateMd should create TOC', async (t) => {
@@ -97,7 +70,7 @@ test('generateMd should create TOC', async (t) => {
 });

 test('should push', async (t) => {
-  await pushNewFile('# title');
+  await pushNewFiles([{ filename: 'README.md', data: '# title' }]);
  t.true(writeFile.calledWith('README.md', '# title'));
  t.true(pull.called);
  t.true(add.calledWith('README.md'));
Author	SHA1	Message	Date
Conventional Changelog Action	0e016f41d3	chore(release): v1.0.40	2021-10-13 22:22:11 +00:00
GitHub Actions	8f9b4ccf8d	fix: fixes while exit condition	2021-10-14 00:20:38 +02:00
GitHub Actions	51da6c20db	feat: paginates using async generator	2021-10-13 23:44:05 +02:00
Conventional Changelog Action	f7a8341bb1	chore(release): v1.0.39	2021-08-31 09:17:17 +00:00
Simone Corsi	c361d6e2b6	Merge pull request #12 from simonecorsi/dependabot/npm_and_yarn/path-parse-1.0.7 build(deps): bump path-parse from 1.0.6 to 1.0.7	2021-08-31 11:16:32 +02:00
dependabot[bot]	ef137b518e	build(deps): bump path-parse from 1.0.6 to 1.0.7 Bumps [path-parse](https://github.com/jbgutierrez/path-parse) from 1.0.6 to 1.0.7. - [Release notes](https://github.com/jbgutierrez/path-parse/releases) - [Commits](https://github.com/jbgutierrez/path-parse/commits/v1.0.7) --- updated-dependencies: - dependency-name: path-parse dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com>	2021-08-12 23:57:13 +00:00
Conventional Changelog Action	3e66d85077	chore(release): v1.0.38	2021-06-23 22:10:04 +00:00
Conventional Changelog Action	16430922c4	chore(release): v1.0.37	2021-06-23 22:09:38 +00:00
Simone Corsi	20b7de127d	Merge pull request #7 from simonecorsi/dependabot/npm_and_yarn/hosted-git-info-2.8.9 build(deps): bump hosted-git-info from 2.8.8 to 2.8.9	2021-06-24 00:09:33 +02:00
Simone Corsi	9def6961a9	Merge pull request #9 from simonecorsi/dependabot/npm_and_yarn/normalize-url-4.5.1 build(deps): bump normalize-url from 4.5.0 to 4.5.1	2021-06-24 00:09:23 +02:00
Simone Corsi	f846c5ee47	Merge pull request #10 from simonecorsi/dependabot/npm_and_yarn/glob-parent-5.1.2 build(deps): bump glob-parent from 5.1.1 to 5.1.2	2021-06-24 00:09:00 +02:00
Simone Corsi	8133f1f32e	Merge branch 'main' into dependabot/npm_and_yarn/normalize-url-4.5.1	2021-06-24 00:08:00 +02:00
Simone Corsi	19e564c928	Merge branch 'main' into dependabot/npm_and_yarn/glob-parent-5.1.2	2021-06-24 00:07:19 +02:00
Conventional Changelog Action	9c0b62271d	chore(release): v1.0.36	2021-06-23 22:05:32 +00:00
GitHub Actions	7ceb9cb783	feat: saving json data for future use	2021-06-24 00:04:22 +02:00
dependabot[bot]	e558e07817	build(deps): bump glob-parent from 5.1.1 to 5.1.2 Bumps [glob-parent](https://github.com/gulpjs/glob-parent) from 5.1.1 to 5.1.2. - [Release notes](https://github.com/gulpjs/glob-parent/releases) - [Changelog](https://github.com/gulpjs/glob-parent/blob/main/CHANGELOG.md) - [Commits](https://github.com/gulpjs/glob-parent/compare/v5.1.1...v5.1.2) --- updated-dependencies: - dependency-name: glob-parent dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com>	2021-06-12 15:27:03 +00:00
dependabot[bot]	202d6fe6c3	build(deps): bump normalize-url from 4.5.0 to 4.5.1 Bumps [normalize-url](https://github.com/sindresorhus/normalize-url) from 4.5.0 to 4.5.1. - [Release notes](https://github.com/sindresorhus/normalize-url/releases) - [Commits](https://github.com/sindresorhus/normalize-url/commits) --- updated-dependencies: - dependency-name: normalize-url dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com>	2021-06-12 03:41:41 +00:00
dependabot[bot]	110a667b31	build(deps): bump hosted-git-info from 2.8.8 to 2.8.9 Bumps [hosted-git-info](https://github.com/npm/hosted-git-info) from 2.8.8 to 2.8.9. - [Release notes](https://github.com/npm/hosted-git-info/releases) - [Changelog](https://github.com/npm/hosted-git-info/blob/v2.8.9/CHANGELOG.md) - [Commits](https://github.com/npm/hosted-git-info/compare/v2.8.8...v2.8.9) Signed-off-by: dependabot[bot] <support@github.com>	2021-05-19 09:24:36 +00:00