HackerNews 자동화 파이프라인 만들기

// Hacker News에서 상위 기사 ID 목록을 가져옴
logMessage("🔄 Fetching new data from HackerNews API...");
const topStoriesRes = await fetch(`${HN_API_BASE}/topstories.json`);
const topStoryIds: number[] = await topStoriesRes.json();
const top100Stories = topStoryIds.slice(0, 100);
 
// 각 기사의 세부 정보를 가져옴 (병렬 요청)
const newsPromises = top100Stories.map(async (id) => {
  const newsRes = await fetch(`${HN_API_BASE}/item/${id}.json`);
  const newsData = await newsRes.json();
  const cleanedTitle = newsData.title ? cleanHNTitle(newsData.title) : null;
 
  return {
    id: generateUUID(newsData.title),
    hnId: newsData.id,
    title: {
      en: cleanedTitle,
      ko: null,
      ja: null,
    },
    type: newsData.type,
    url: newsData.url ?? null,
    score: newsData.score ?? null,
    by: newsData.by ?? null,
    time: newsData.time ?? null,
    content: newsData.text ?? null,
    summary: {
      en: null,
      ko: null,
      ja: null,
    },
  };
});

// 일반적인 경우
const selectors = [
  "article",
  "div#content",
  "div.content",
  "div#post-content",
  "div.content-area",
  "div#main",
  "div.main",
  "div.prose",
  "div.entry",
  "div.bodycopy",
  "div.node__content",
  "div.essay__content",
  "main",
];
 
for (const selector of selectors) {
  const el = document.querySelector(selector);
  if (el) return el.textContent || "";
}
 
// <section> 대체 방법
const sections = document.querySelectorAll("section");
if (sections.length > 0) {
  return Array.from(sections)
    .map((sec) => sec.textContent?.trim() || "")
    .join("\n\n");
}

// 이진 탐색
while (start <= end) {
  const mid = Math.floor((start + end) / 2);
  const currentText = text.slice(0, mid);
  const currentTokenCount = await countTokens(currentText);
 
  console.log(`Words (mid): ${mid}, Tokens: ${currentTokenCount}`);
 
  if (currentTokenCount === maxTokens) {
    return currentText;
  } else if (currentTokenCount <= maxTokens) {
    start = mid + 1;
  } else {
    end = mid;
  }
}

for (const [idx, item] of toFetch.entries()) {
  fetchQueue.add(async () => {
    try {
      logMessage(`[${idx + 1}/${toFetch.length}] Fetching: ${item.id}...`);
      let content = null;
 
      if (item.url.includes(".pdf")) {
        content = await fetchPdfContent(item.url);
        logMessage(`📄 PDF content extracted for ${item.url}`);
      } else {
        content = await fetchContent(item.url);
        logMessage(`🌐 Smart content fetched for ${item.url}`);
      }
 
      if (content) {
        content = await sliceTextByTokens(content, 15000);
        logMessage(`📄 Sliced content (up to 15000 tokens)`);
      }
 
      // Redis를 사용하여 콘텐츠를 임시 저장
      if (content) {
        await redis.set(`content:${item.id}`, content, "EX", 60 * 60 * 24);
      }
      logMessage(`[${idx + 1}/${toFetch.length}] ✅ Done: ${item.id}`);
    } catch (error) {
      logMessage(
        `[${idx + 1}/${toFetch.length}] ❌ Error: ${item.id} (${error})`,
      );
    }
  });
}

// 요약 번역
if (
  item.summary &&
  item.summary.en &&
  (!item.summary.ja || item.summary.ja === "")
) {
  translateQueue.add(async () => {
    try {
      logMessage(
        `[ja][summary][${idx + 1}/${toTranslateJa.length}] Translating summary: ${item.id}...`,
      );
      const translated = await translate(item.summary.en, "ja", "content");
      await redis.set(`ja:summary:${item.id}`, translated, "EX", 60 * 60 * 24);
      logMessage(
        `[ja][summary][${idx + 1}/${toTranslateJa.length}] ✅ Done: ${item.id}`,
      );
    } catch (error) {
      logMessage(
        `[ja][summary][${idx + 1}/${toTranslateJa.length}] ❌ Error: ${item.id} (${error})`,
      );
    }
  });
}

const extractedKeywordsString = await keywords(date);
const extractedKeywords = JSON.parse(extractedKeywordsString);
 
const siteUrl = process.env.NEXT_PUBLIC_BASE_URL || "https://mogumogu.dev";
const res = await fetch(`${siteUrl}/prompts/picture-style.md`);
if (!res.ok) throw new Error("Style prompt fetch failed");
const stylePrompt = await res.text();
const styleKeywords = parseStylePrompt(stylePrompt);
 
const flattenedKeywords = {
  whatisInTheImage: flattenToArray(extractedKeywords.whatIsInTheImage),
  background: flattenToArray(extractedKeywords.background),
};
 
const fullPrompt = {
  style: styleKeywords.style,
  mood: styleKeywords.mood,
  perspective: styleKeywords.perspective,
  colors: styleKeywords.colors,
  additionalEffects: styleKeywords.additionalEffects,
  whatisInTheImage: flattenedKeywords.whatisInTheImage,
  background: flattenedKeywords.background,
};

- name: Replace Redis value in .env.prod using Ansible
  uses: dawidd6/action-ansible-playbook@v2
  with:
      playbook: ansible/replace-env.yml
      directory: .
      inventory: |
      [web]
      ec2-snstance ansible_host=${{ secrets.AWS_EC2_HOST }} ansible_user=ubuntu ansible_ssh_private_key_file=/tmp/ci-key.pem
  env:
      REDIS_HOST: ${{ env.REDIS_HOST }}
      REDIS_PORT: ${{ env.REDIS_PORT }}

- name: Summarize HackerNews content
  run: |
    RESPONSE=$(curl -s -X POST https://mogumogu.dev/api/hackernews/summarize \
    -H "Content-Type: application/json" \
    -H "Authorization: Bearer ${{ secrets.HACKERNEWS_API_KEY }}")
    echo "$RESPONSE"
    SUMMARIZE_TOTAL=$(echo "$RESPONSE" | jq '.total')
    echo "SUMMARIZE_TOTAL=$SUMMARIZE_TOTAL" >> $GITHUB_ENV

- name: Summarize HackerNews content - Flush
  id: summarize-flush
  run: |
    for i in {1..5}; do
    RESPONSE=$(curl -s -X POST https://mogumogu.dev/api/hackernews/flush \
        -H "Content-Type: application/json" \
        -H "Authorization: Bearer ${{ secrets.HACKERNEWS_API_KEY }}" \
        -d "{\"type\": \"summarize\", \"total\": $SUMMARIZE_TOTAL}")
    echo "$RESPONSE"
    FLUSHED=$(echo "$RESPONSE" | jq '.flushed')
    if [ "$FLUSHED" -ge 1 ]; then
        echo "Summarize flush succeeded!"
        exit 0
    fi
    echo "Summarize flush not ready, retrying in 60s..."
    sleep 60
    done
    echo "Summarize flush failed after 5 attempts"
    exit 0