Feat/crwal youtube

.gitignore

@@ -39,5 +39,5 @@ yarn-error.log*
 *.pem
 
 # Crawling
-**/logs/*.txt
+**/log/*.txt
 

apps/web/next.config.ts

@@ -1,7 +1,14 @@
+import withBundleAnalyzer from '@next/bundle-analyzer';
 import type { NextConfig } from 'next';
 
 const nextConfig: NextConfig = {
   /* config options here */
 };
 
-export default nextConfig;
+const withBundle = withBundleAnalyzer({
+  // enabled: process.env.ANALYZE === 'true',
+  enabled: true,
+  openAnalyzer: true,
+});
+
+export default withBundle(nextConfig);

apps/web/package.json

@@ -15,6 +15,7 @@
     "@dnd-kit/modifiers": "^9.0.0",
     "@dnd-kit/sortable": "^10.0.0",
     "@dnd-kit/utilities": "^3.2.2",
+    "@next/bundle-analyzer": "^15.3.2",
     "@radix-ui/react-checkbox": "^1.1.5",
     "@radix-ui/react-dialog": "^1.1.6",
     "@radix-ui/react-dropdown-menu": "^2.1.6",

packages/crawling/package.json

@@ -6,6 +6,12 @@
   "exports": {
     ".": "./src/index.js"
   },
+  "scripts": {
+    "dev-ky-open": "tsx src/findKYByOpen.ts",
+    "dev-ky-youtube": "tsx src/crawlYoutube.ts",
+    "lint": "eslint . --ext .ts,.js",
+    "test": "vitest run"
+  },
   "dependencies": {
     "@repo/open-api": "workspace:*",
     "@supabase/supabase-js": "^2.49.1",

packages/crawling/src/crawlYoutube.ts

@@ -5,57 +5,102 @@ import { Song } from "./types";
 import { updateDataLog } from "./logData";
 import { updateKYDB } from "./supabase/updateDB";
 
+const stackData: Song[] = [];
+const totalData: Song[] = [];
+
+// process.on("SIGINT", async () => {
+//   console.log("프로세스가 종료됩니다. 지금까지의 데이터를 업데이트 중...");
+//   console.log("stackData : ", stackData.length);
+//   const result = await updateKYDB(stackData);
+
+//   console.log(result);
+//   console.log("프로세스가 종료됩니다. 로그를 기록 중...");
+
+//   await Promise.all([
+//     updateDataLog(totalData, "log/crawlYodutubeSuccess.txt"),
+//     updateDataLog(failedCase, "log/crawlYoutubeFailed.txt"),
+//   ]);
+
+//   console.log("로그 기록 완료.");
+// });
+
 const browser = await puppeteer.launch();
 const page = await browser.newPage();
 
 const baseUrl = "https://www.youtube.com/@KARAOKEKY/search";
 
-async function scrapeSongNumber(query: string) {
+const scrapeSongNumber = async (query: string) => {
   const searchUrl = `${baseUrl}?query=${encodeURIComponent(query)}`;
-  await page.goto(searchUrl);
+
+  // page.goto의 waitUntil 문제였음!
+  await page.goto(searchUrl, {
+    waitUntil: "networkidle2",
+    timeout: 0,
+  });
 
   const html = await page.content();
   const $ = cheerio.load(html);
+
   // id contents 의 첫번째  ytd-item-section-renderer 찾기
-  const firstItem = $("#contents ytd-item-section-renderer").first();
+  // const firstItem = $("#contents ytd-item-section-renderer").first();
+
+  const firstItem = $("ytd-video-renderer").first();
+
   // yt-formatted-string 찾기
   const title = firstItem.find("yt-formatted-string").first().text().trim();
 
   const karaokeNumber = extractKaraokeNumber(title);
 
-  // await browser.close();
-
   return karaokeNumber;
-}
+};
 
-function extractKaraokeNumber(title: string) {
+const extractKaraokeNumber = (title: string) => {
   // KY. 찾고 ) 가 올때까지 찾기
-  const matchResult = title.match(/KY\.\s*(\d{3,5})\)/);
+  const matchResult = title.match(/KY\.\s*(\d{2,5})\)/);
   const karaokeNumber = matchResult ? matchResult[1] : null;
   return karaokeNumber;
-}
+};
+
+const refreshData = async () => {
+  console.log("refreshData");
+  const result = await updateKYDB(stackData);
+
+  updateDataLog(result.success, "log/crawlYoutubeSuccess.txt");
+  updateDataLog(result.failed, "log/crawlYoutubeFailed.txt");
 
+  stackData.length = 0; // stackData 초기화
+};
 // 사용
 
 const data = await getKYNULLDB();
-console.log("getKYNULLDB : ", data);
-const resultData: Song[] = [];
+console.log("getKYNULLDB : ", data.length);
 let index = 0;
 
 for (const song of data) {
-  const query = song.title + " - " + song.artist;
+  if (stackData.length > 100) {
+    refreshData();
+  }
+  const query = song.title + "-" + song.artist;
+  console.log(song.title, " - ", song.artist);
   const result = await scrapeSongNumber(query);
   if (result) {
-    resultData.push({ ...song, num_ky: result });
+    console.log("success : ", result);
+    stackData.push({ ...song, num_ky: result });
+    totalData.push({ ...song, num_ky: result });
   }
   index++;
   console.log("scrapeSongNumber : ", index);
+  console.log("stackData : ", stackData.length);
 }
 
-console.log("resultData : ", resultData.length);
-const result = await updateKYDB(resultData);
-
-console.log(result);
+console.log("totalData : ", totalData.length);
+// const result = await updateKYDB(totalData);
+const result = await updateKYDB(stackData);
 
 updateDataLog(result.success, "log/crawlYoutubeSuccess.txt");
 updateDataLog(result.failed, "log/crawlYoutubeFailed.txt");
+
+// 5.13 1차 시도
+// 5000개 중 3507개 성공, 총 18906개 등록
+
+// 5.13 2차 시도

packages/crawling/src/findKYByOpen.ts

@@ -69,7 +69,7 @@ for (const song of kyNullData) {
 // 6079개 업데이트
 
 // 2차 시도
-// 15065개 업데이트, 제목 가수 이름 일치 이슈
+// 15065개 업데이트, 제목 가수 이름 불일치 이슈
 
 console.log(`
     총 ${kyNullData.length}곡 중:

packages/crawling/src/logData.ts

@@ -16,6 +16,12 @@ export function updateDataLog<T>(unknownData: T[] | T, filename: string) {
   });
 
   const logPath = path.join(filename);
+  const logDir = path.dirname(logPath); // 디렉터리 경로 추출
+
+  // 디렉터리가 없으면 생성
+  if (!fs.existsSync(logDir)) {
+    fs.mkdirSync(logDir, { recursive: true });
+  }
 
   if (unknownData instanceof Array) {
     // 로그 문자열 생성

packages/crawling/src/supabase/getDB.ts

@@ -32,26 +32,35 @@ export async function getJapaneseDB() {
   return hasJapaneseData;
 }
 
-export async function getKYNULLDB() {
+export async function getKYNULLDB(max: number = 50000) {
   const supabase = getClient();
 
   // artist 정렬
+  // const { data, error } = await supabase
+  //   .from("songs")
+  //   .select("id, title, artist, num_tj, num_ky")
+  //   .order("title", { ascending: true });
+
   const { data, error } = await supabase
     .from("songs")
     .select("id, title, artist, num_tj, num_ky")
-    .order("title", { ascending: true });
+    .is("num_ky", null) // num_ky가 null인 데이터만 가져옴
+    .order("title", { ascending: false })
+    .limit(max); // Supabase 쿼리 안에서의 한계를 넘을 수는 없음
 
   if (error) throw error;
 
   console.log("data", data.length);
 
-  const isKYNULLData: Song[] = [];
+  return data;
 
-  data.forEach((song) => {
-    if (song.num_ky === null) {
-      isKYNULLData.push(song);
-    }
-  });
+  // const isKYNULLData: Song[] = [];
+
+  // data.forEach((song) => {
+  //   if (song.num_ky === null) {
+  //     isKYNULLData.push(song);
+  //   }
+  // });
 
-  return isKYNULLData;
+  // return isKYNULLData.slice(0, max);
 }