fetch longcat tutorials.ts

/**
 * 优化版：专注采集 LongCat Video/Image AI 工具的教程视频
 * 筛选标准：
 * - 时长 > 3 分钟
 * - 包含教程关键词
 * - 排除游戏、娱乐类视频
 *
 * 运行方式: pnpm tsx "SEO/Scripts for Youtube/fetch-longcat-tutorials.ts"
 */

import fs from 'node:fs';
import path from 'node:path';
import { config } from 'dotenv';
import { getSubtitles } from 'youtube-caption-extractor';

// 加载环境变量
config({ path: '.env.development' });

// YouTube API 配置
const YOUTUBE_API_KEY = process.env.YOUTUBE_API_KEY;
const YOUTUBE_API_BASE = 'https://www.googleapis.com/youtube/v3';

// 延迟函数
function delay(ms: number) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

// 解析ISO 8601时长为秒数
function parseDurationToSeconds(isoDuration: string): number {
  const match = isoDuration.match(/PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?/);
  if (!match) return 0;

  const hours = Number.parseInt(match[1] || '0', 10);
  const minutes = Number.parseInt(match[2] || '0', 10);
  const seconds = Number.parseInt(match[3] || '0', 10);

  return hours * 3600 + minutes * 60 + seconds;
}

// 格式化时长
function formatDuration(isoDuration: string): string {
  const match = isoDuration.match(/PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?/);
  if (!match) return isoDuration;

  const hours = match[1] ? `${match[1]}:` : '';
  const minutes = match[2] ? match[2].padStart(hours ? 2 : 1, '0') : '00';
  const seconds = match[3] ? match[3].padStart(2, '0') : '00';

  return `${hours}${minutes}:${seconds}`;
}

// 格式化播放量
function formatViewCount(count: string): string {
  const num = Number.parseInt(count, 10);
  if (num >= 1000000) return `${(num / 1000000).toFixed(1)}M`;
  if (num >= 1000) return `${(num / 1000).toFixed(1)}K`;
  return count;
}

// 检查视频是否相关（排除游戏、娱乐类）
function isRelevantVideo(title: string, description: string): boolean {
  const lowerTitle = title.toLowerCase();
  const lowerDesc = description.toLowerCase();

  // 排除关键词（游戏、纯娱乐）
  const excludeKeywords = [
    'fancade',
    'game level',
    'gaming',
    'speed level',
    'cat dance',
    'funny cat',
    'cute cat',
    'meme',
    'compilation',
    'shorts',
  ];

  for (const keyword of excludeKeywords) {
    if (lowerTitle.includes(keyword)) {
      return false;
    }
  }

  // 必须包含的关键词之一（AI工具相关）
  const requiredKeywords = [
    'longcat',
    'comfyui',
    'tutorial',
    'workflow',
    'guide',
    'how to',
    'avatar',
    'ai video',
    'image generator',
  ];

  const hasRequired = requiredKeywords.some(
    (keyword) => lowerTitle.includes(keyword) || lowerDesc.includes(keyword)
  );

  return hasRequired;
}

// 搜索视频（支持分页和时长过滤）
async function searchVideos(keyword: string, maxResults = 50) {
  console.log(`\n🔍 搜索关键词: "${keyword}"`);
  console.log(`📊 最大结果数: ${maxResults} (筛选后)\n`);

  if (!YOUTUBE_API_KEY) {
    throw new Error('YOUTUBE_API_KEY 环境变量未设置');
  }

  let allItems: any[] = [];
  let pageToken: string | undefined = undefined;
  let pageCount = 0;
  let totalFetched = 0;

  // 循环获取直到达到目标数量或无更多结果
  while (allItems.length < maxResults && pageCount < 10) {
    pageCount++;
    const perPageResults = 50; // YouTube API 每次最多50个

    console.log(`📄 获取第 ${pageCount} 页 (每页 ${perPageResults} 个)...`);

    const params: any = {
      part: 'snippet',
      q: keyword,
      type: 'video',
      maxResults: perPageResults.toString(),
      order: 'relevance',
      videoDuration: 'medium', // 4-20分钟
      key: YOUTUBE_API_KEY,
    };

    if (pageToken) {
      params.pageToken = pageToken;
    }

    const searchResponse = await fetch(
      `${YOUTUBE_API_BASE}/search?${new URLSearchParams(params)}`
    );

    if (!searchResponse.ok) {
      const error = await searchResponse.json();
      throw new Error(`YouTube 搜索失败: ${JSON.stringify(error)}`);
    }

    const searchData = await searchResponse.json();

    if (!searchData.items || searchData.items.length === 0) {
      console.log('⚠️  没有更多视频了');
      break;
    }

    totalFetched += searchData.items.length;
    allItems = allItems.concat(searchData.items);
    console.log(`   ✓ 累计获取 ${totalFetched} 个视频 (筛选前)`);

    if (!searchData.nextPageToken) {
      console.log('✅ 已获取所有结果（没有更多页面）\n');
      break;
    }

    pageToken = searchData.nextPageToken;
    await delay(1000);
  }

  if (allItems.length === 0) {
    return { videos: [], totalResults: 0 };
  }

  // 批量获取视频详细信息
  console.log(`\n📊 获取视频详情和筛选...`);
  const allVideosData: any[] = [];

  for (let i = 0; i < allItems.length; i += 50) {
    const batch = allItems.slice(i, i + 50);
    const videoIds = batch.map((item: any) => item.id.videoId).join(',');

    console.log(
      `   处理 ${i + 1}-${Math.min(i + 50, allItems.length)}/${allItems.length}...`
    );

    const videosResponse = await fetch(
      `${YOUTUBE_API_BASE}/videos?${new URLSearchParams({
        part: 'snippet,contentDetails,statistics',
        id: videoIds,
        key: YOUTUBE_API_KEY,
      })}`
    );

    if (!videosResponse.ok) {
      const error = await videosResponse.json();
      throw new Error(`获取视频详情失败: ${JSON.stringify(error)}`);
    }

    const videosData = await videosResponse.json();
    allVideosData.push(...videosData.items);

    if (i + 50 < allItems.length) {
      await delay(1000);
    }
  }

  console.log('✅ 获取所有视频详情完成\n');

  // 筛选和格式化视频数据
  const videos = allVideosData
    .filter((item: any) => {
      // 1. 时长筛选：> 3 分钟 (180秒)
      const durationSeconds = parseDurationToSeconds(
        item.contentDetails.duration
      );
      if (durationSeconds < 180) {
        return false;
      }

      // 2. 内容相关性筛选
      if (!isRelevantVideo(item.snippet.title, item.snippet.description)) {
        return false;
      }

      return true;
    })
    .map((item: any) => ({
      id: item.id,
      title: item.snippet.title,
      channelTitle: item.snippet.channelTitle,
      channelId: item.snippet.channelId,
      publishedAt: item.snippet.publishedAt,
      description: item.snippet.description,
      thumbnailUrl:
        item.snippet.thumbnails.high?.url ||
        item.snippet.thumbnails.medium?.url,
      duration: formatDuration(item.contentDetails.duration),
      durationSeconds: parseDurationToSeconds(item.contentDetails.duration),
      viewCount: formatViewCount(item.statistics.viewCount || '0'),
      viewCountRaw: Number.parseInt(item.statistics.viewCount || '0', 10),
      likeCount: Number.parseInt(item.statistics.likeCount || '0', 10),
      commentCount: Number.parseInt(item.statistics.commentCount || '0', 10),
      url: `https://www.youtube.com/watch?v=${item.id}`,
    }));

  console.log(`🎯 筛选结果: ${videos.length} 个符合条件的视频\n`);

  return {
    keyword,
    searchDate: new Date().toISOString(),
    totalResults: videos.length,
    videos,
  };
}

// 提取单个视频脚本
async function extractScript(videoId: string) {
  try {
    console.log(`  → 提取视频: ${videoId}`);

    const captions = await getSubtitles({ videoID: videoId });

    let script = '';
    let hasTranscript = false;

    if (captions && captions.length > 0) {
      script = captions
        .map((item) => item.text)
        .join(' ')
        .replace(/\s+/g, ' ')
        .trim();
      hasTranscript = true;
    }

    console.log(
      `  ✓ ${videoId}: ${hasTranscript ? `成功 (${script.length} 字符)` : '无字幕'}`
    );

    return {
      videoId,
      success: true,
      hasTranscript,
      script,
      scriptLength: script.length,
      extractedAt: new Date().toISOString(),
    };
  } catch (error: any) {
    console.log(`  ✗ ${videoId}: 失败 - ${error.message}`);
    return {
      videoId,
      success: false,
      hasTranscript: false,
      script: '',
      error: error.message,
    };
  }
}

// 批量提取脚本
async function batchExtractScripts(videoIds: string[]) {
  console.log(`\n📝 开始批量提取脚本 (${videoIds.length} 个视频)\n`);

  const results = [];

  for (let i = 0; i < videoIds.length; i++) {
    const videoId = videoIds[i];
    console.log(`[${i + 1}/${videoIds.length}]`);

    const result = await extractScript(videoId);
    results.push(result);

    if (i < videoIds.length - 1) {
      await delay(1000);
    }
  }

  const successCount = results.filter((r) => r.success).length;
  const hasTranscriptCount = results.filter((r) => r.hasTranscript).length;
  const failedCount = results.filter((r) => !r.success).length;

  console.log('\n✅ 提取完成:');
  console.log(`   成功: ${successCount} 个`);
  console.log(`   有字幕: ${hasTranscriptCount} 个`);
  console.log(`   失败: ${failedCount} 个\n`);

  return results;
}

// 保存结果到文件
function saveResults(data: any, filename: string, targetDir?: string) {
  const dataDir = targetDir ?? path.join(process.cwd(), 'data/seo');

  if (!fs.existsSync(dataDir)) {
    fs.mkdirSync(dataDir, { recursive: true });
  }

  const filePath = path.join(dataDir, filename);
  fs.writeFileSync(filePath, JSON.stringify(data, null, 2), 'utf-8');

  console.log(`💾 结果已保存到: ${filePath}`);
  console.log(
    `📦 文件大小: ${(fs.statSync(filePath).size / 1024).toFixed(2)} KB\n`
  );

  return filePath;
}

// 主函数
async function main() {
  try {
    console.log('🚀 开始获取 LongCat 教程视频 (优化版)\n');
    console.log('='.repeat(60));

    // 优化的教程类关键词
    const keywords = [
      'longcat video tutorial',
      'longcat comfyui workflow',
      'longcat avatar tutorial',
      'longcat video guide',
      'how to use longcat video',
      'longcat image tutorial',
      'comfyui longcat',
      'longcat ai video',
    ];

    console.log(`\n📋 搜索策略: 使用 ${keywords.length} 个教程类关键词`);
    console.log(`🎯 筛选标准: 时长 > 3 分钟, 排除游戏/娱乐类\n`);

    const allVideos: any[] = [];
    const videoIdsSet = new Set<string>();

    for (const keyword of keywords) {
      console.log(`\n${'='.repeat(60)}`);
      const searchResult = await searchVideos(keyword, 50);

      let newVideos = 0;
      for (const video of searchResult.videos) {
        if (!videoIdsSet.has(video.id)) {
          videoIdsSet.add(video.id);
          allVideos.push(video);
          newVideos++;
        }
      }

      console.log(
        `   ℹ️  新增 ${newVideos} 个不重复视频，累计 ${allVideos.length} 个`
      );

      await delay(2000);
    }

    console.log(`\n${'='.repeat(60)}`);
    console.log(
      `\n✅ 所有关键词搜索完成，共获取 ${allVideos.length} 个教程视频\n`
    );

    if (allVideos.length === 0) {
      console.log('没有找到相关视频，程序结束。');
      return;
    }

    // 批量提取脚本
    const videoIds = allVideos.map((v) => v.id);
    const extractResults = await batchExtractScripts(videoIds);

    // 合并结果
    const completeData = {
      keywords: keywords,
      searchDate: new Date().toISOString(),
      totalVideos: allVideos.length,
      videos: allVideos.map((video) => {
        const extractResult = extractResults.find(
          (r) => r.videoId === video.id
        );
        return {
          ...video,
          script: extractResult?.script || '',
          hasTranscript: extractResult?.hasTranscript || false,
          scriptLength: extractResult?.scriptLength || 0,
          extractedAt: extractResult?.extractedAt || null,
          extractError: extractResult?.error || null,
        };
      }),
      statistics: {
        totalVideos: allVideos.length,
        successfulExtracts: extractResults.filter((r) => r.success).length,
        videosWithTranscript: extractResults.filter((r) => r.hasTranscript)
          .length,
        failedExtracts: extractResults.filter((r) => !r.success).length,
        highPriority: allVideos.filter((v) => v.viewCountRaw > 10000).length,
        mediumPriority: allVideos.filter(
          (v) => v.viewCountRaw > 1000 && v.viewCountRaw <= 10000
        ).length,
      },
      exportedAt: new Date().toISOString(),
    };

    // 保存结果
    console.log('='.repeat(60));
    saveResults(
      completeData,
      `longcat-tutorials-${Date.now()}.json`,
      path.join(process.cwd(), 'data/seo')
    );
    saveResults(
      completeData,
      'longcat-tutorials.json',
      path.join(process.cwd(), 'SEO/Videos')
    );

    // 打印统计信息
    console.log('📊 统计信息:');
    console.log(`   总视频数: ${completeData.statistics.totalVideos}`);
    console.log(`   成功提取: ${completeData.statistics.successfulExtracts}`);
    console.log(`   有字幕: ${completeData.statistics.videosWithTranscript}`);
    console.log(`   失败: ${completeData.statistics.failedExtracts}`);
    console.log(
      `   高优先级 (>10K播放): ${completeData.statistics.highPriority}`
    );
    console.log(
      `   中优先级 (1K-10K): ${completeData.statistics.mediumPriority}`
    );

    // Top 10
    console.log('\n📈 播放量 Top 10 视频:');
    const topVideos = completeData.videos
      .sort((a, b) => b.viewCountRaw - a.viewCountRaw)
      .slice(0, 10);

    for (let i = 0; i < topVideos.length; i++) {
      const video = topVideos[i];
      console.log(
        `   ${i + 1}. ${video.title.substring(0, 80)}${video.title.length > 80 ? '...' : ''}`
      );
      console.log(`      播放量: ${video.viewCount} | ${video.url}`);
    }

    console.log('\n✨ 全部完成！');
  } catch (error) {
    console.error('\n❌ 错误:', error);
    process.exit(1);
  }
}

// 运行
main();

本文档为站内渲染。原始文件本地路径：saas/source/seo-llm/raw-seo知识库-TXT整理-seo方法论-seo-knowledge-base-Scripts-for-Youtu-999970.txt（仅本地保留，不入库不部署）