diff --git a/src/helpers.ts b/src/helpers.ts index 3687671..839fd15 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -1,3 +1,4 @@ +import fs from 'fs'; import ejs from 'ejs'; import * as core from '@actions/core'; import remark from 'remark'; @@ -8,9 +9,10 @@ import GithubApi from './api'; import link from './link'; import git from './git'; -import type { PaginationLink, ApiGetStarResponse } from './types'; +import type { PaginationLink, ApiGetStarResponse, Stars } from './types'; -import fs from 'fs'; +export const REPO_USERNAME = process.env.GITHUB_REPOSITORY?.split('/')[0]; +export const API_STARRED_URL = `${process.env.GITHUB_API_URL}/users/${REPO_USERNAME}/starred`; const fsp = fs.promises; @@ -18,10 +20,6 @@ export function wait(time = 200): Promise { return new Promise((resolve) => setTimeout(resolve, time)); } -export function isLastPage(links: PaginationLink): boolean { - return links.next === links.last; -} - export async function renderer( data: { [key: string]: any }, templateString = MD_TEMPLATE @@ -34,32 +32,43 @@ export async function renderer( } } -export async function apiGetStar(url: string): Promise { - const { headers, body }: any = await GithubApi.get(url); - return { - data: body, - links: link.parse(headers.link).refs.reduce( - (acc, val) => ({ - ...acc, - [val.rel]: val.uri, - }), - {} - ), - }; +export function getNextPage(links: PaginationLink[]): string | null { + const link = links.find((l) => l.rel === 'next'); + if (!link) return null; + const match = link.uri.match(/page=([0-9]*)/); + if (!match) return null; + return match[1]; } -export const REPO_USERNAME = process.env.GITHUB_REPOSITORY?.split('/')[0]; -export const API_STARRED_URL = `${process.env.GITHUB_API_URL}/users/${REPO_USERNAME}/starred`; +async function* paginateStars(url: string): AsyncGenerator { + let nextPage: string | null = '1'; + while (nextPage) { + try { + const { headers, body } = await GithubApi.get(url, { + searchParams: { + page: nextPage, + }, + }); + console.log('body, typeof body :>> ', body, typeof body); + yield (body as unknown) as Stars; + nextPage = getNextPage(link.parse(headers.link).refs); + console.log('sleeping 1second to avoid rate-limit'); + await wait(1000); // avoid limits + } catch (e) { + console.error(e); + break; + } + } +} -let links: PaginationLink = { - next: API_STARRED_URL, - last: undefined, -}; -export async function paginate(): Promise { - if (isLastPage(links)) return null; - const r = await apiGetStar(links.next); - links = r.links; - return r; +export async function apiGetStar( + url: string = API_STARRED_URL +): Promise { + let data: Stars[] = []; + for await (const stars of paginateStars(url)) { + data = data.concat(stars); + } + return (data as unknown) as Stars; } export function generateMd(data: string): Promise { diff --git a/src/index.ts b/src/index.ts index a120f63..abc0552 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,26 +1,18 @@ import * as core from '@actions/core'; -import { data } from 'remark'; import { renderer, - paginate, REPO_USERNAME, generateMd, pushNewFiles, MARKDOWN_FILENAME, + apiGetStar, } from './helpers'; import type { SortedLanguageList, Stars, Star } from './types'; export async function main(): Promise { - let results: Stars = []; - - while (true) { - // sorry. - const r = await paginate(); - if (!r || r === null) break; - results = results.concat(r.data); - } + const results: Stars = await apiGetStar(); const sortedByLanguages = results.reduce( (acc: SortedLanguageList, val: Star) => { diff --git a/src/types.ts b/src/types.ts index 849d4a4..92a32d2 100644 --- a/src/types.ts +++ b/src/types.ts @@ -5,14 +5,11 @@ export type SortedLanguageList = { }; export type PaginationLink = { - next: string; - last: string | undefined | null; + uri: string; + rel: 'next' | 'last' | 'prev' | 'first'; }; export type Stars = Endpoints['GET /user/starred']['response']['data']; export type Star = Stars[number] | { language: string }; -export type ApiGetStarResponse = { - links: PaginationLink; - data: Stars; -}; +export type ApiGetStarResponse = Stars;