feat: paginates using async generator

src/helpers.ts

@@ -1,3 +1,4 @@
+import fs from 'fs';
 import ejs from 'ejs';
 import * as core from '@actions/core';
 import remark from 'remark';
@@ -8,9 +9,10 @@ import GithubApi from './api';
 import link from './link';
 import git from './git';
 
-import type { PaginationLink, ApiGetStarResponse } from './types';
+import type { PaginationLink, ApiGetStarResponse, Stars } from './types';
 
-import fs from 'fs';
+export const REPO_USERNAME = process.env.GITHUB_REPOSITORY?.split('/')[0];
+export const API_STARRED_URL = `${process.env.GITHUB_API_URL}/users/${REPO_USERNAME}/starred`;
 
 const fsp = fs.promises;
 
@@ -18,10 +20,6 @@ export function wait(time = 200): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, time));
 }
 
-export function isLastPage(links: PaginationLink): boolean {
-  return links.next === links.last;
-}
-
 export async function renderer(
   data: { [key: string]: any },
   templateString = MD_TEMPLATE
@@ -34,32 +32,43 @@ export async function renderer(
   }
 }
 
-export async function apiGetStar(url: string): Promise<ApiGetStarResponse> {
-  const { headers, body }: any = await GithubApi.get(url);
-  return {
-    data: body,
-    links: link.parse(headers.link).refs.reduce(
-      (acc, val) => ({
-        ...acc,
-        [val.rel]: val.uri,
-      }),
-      {}
-    ),
-  };
+export function getNextPage(links: PaginationLink[]): string | null {
+  const link = links.find((l) => l.rel === 'next');
+  if (!link) return null;
+  const match = link.uri.match(/page=([0-9]*)/);
+  if (!match) return null;
+  return match[1];
 }
 
-export const REPO_USERNAME = process.env.GITHUB_REPOSITORY?.split('/')[0];
-export const API_STARRED_URL = `${process.env.GITHUB_API_URL}/users/${REPO_USERNAME}/starred`;
+async function* paginateStars(url: string): AsyncGenerator<Stars> {
+  let nextPage: string | null = '1';
+  while (nextPage) {
+    try {
+      const { headers, body } = await GithubApi.get(url, {
+        searchParams: {
+          page: nextPage,
+        },
+      });
+      console.log('body, typeof body :>> ', body, typeof body);
+      yield (body as unknown) as Stars;
+      nextPage = getNextPage(link.parse(headers.link).refs);
+      console.log('sleeping 1second to avoid rate-limit');
+      await wait(1000); // avoid limits
+    } catch (e) {
+      console.error(e);
+      break;
+    }
+  }
+}
 
-let links: PaginationLink = {
-  next: API_STARRED_URL,
-  last: undefined,
-};
-export async function paginate(): Promise<ApiGetStarResponse | null> {
-  if (isLastPage(links)) return null;
-  const r = await apiGetStar(links.next);
-  links = r.links;
-  return r;
+export async function apiGetStar(
+  url: string = API_STARRED_URL
+): Promise<ApiGetStarResponse> {
+  let data: Stars[] = [];
+  for await (const stars of paginateStars(url)) {
+    data = data.concat(stars);
+  }
+  return (data as unknown) as Stars;
 }
 
 export function generateMd(data: string): Promise<string> {

src/index.ts

@@ -1,26 +1,18 @@
 import * as core from '@actions/core';
-import { data } from 'remark';
 
 import {
   renderer,
-  paginate,
   REPO_USERNAME,
   generateMd,
   pushNewFiles,
   MARKDOWN_FILENAME,
+  apiGetStar,
 } from './helpers';
 
 import type { SortedLanguageList, Stars, Star } from './types';
 
 export async function main(): Promise<any> {
-  let results: Stars = [];
-
-  while (true) {
-    // sorry.
-    const r = await paginate();
-    if (!r || r === null) break;
-    results = results.concat(r.data);
-  }
+  const results: Stars = await apiGetStar();
 
   const sortedByLanguages = results.reduce(
     (acc: SortedLanguageList, val: Star) => {

src/types.ts

@@ -5,14 +5,11 @@ export type SortedLanguageList = {
 };
 
 export type PaginationLink = {
-  next: string;
-  last: string | undefined | null;
+  uri: string;
+  rel: 'next' | 'last' | 'prev' | 'first';
 };
 
 export type Stars = Endpoints['GET /user/starred']['response']['data'];
 export type Star = Stars[number] | { language: string };
 
-export type ApiGetStarResponse = {
-  links: PaginationLink;
-  data: Stars;
-};
+export type ApiGetStarResponse = Stars;