boc/quixzoom-video-pipeline/discovery/youtube-discovery.js

/**
 * QUIXZOOM Video Pipeline — YouTube Discovery Module
 * 
 * Söker efter och indexerar city walking videos från YouTube.
 * Respekterar upphovsrätt — endast Creative Commons eller egna inspelningar.
 * 
 * Teknik: Node.js, YouTube Data API v3, Redis
 */

const { google } = require('googleapis');
const Redis = require('ioredis');

// Konfiguration
const YOUTUBE_API_KEY = process.env.YOUTUBE_API_KEY;
const redis = new Redis({
  host: process.env.REDIS_HOST || 'localhost',
  port: process.env.REDIS_PORT || 6379,
});

const youtube = google.youtube({
  version: 'v3',
  auth: YOUTUBE_API_KEY,
});

/**
 * Söktermer för city walking videos
 */
const SEARCH_QUERIES = [
  '4K City Walk',
  'Walking Tour',
  'Street Walk',
  'Bangkok Walk',
  'Tokyo Walk',
  'London Walk',
  'Walking Downtown',
  'Night Walk',
  'Walking Tour 4K',
  'POV Walk',
  'City Walking Tour',
  'Urban Exploration Walk',
  'Beach Walk 4K',
  'Cycling Tour City',
  'Dashcam City Drive',
];

/**
 * Stadsspecifika söktermer
 */
const CITY_QUERIES = [
  'Bangkok Walking Tour',
  'Tokyo Walking Tour',
  'London Walking Tour',
  'Paris Walking Tour',
  'Berlin Walking Tour',
  'New York Walking Tour',
  'Dubai Walking Tour',
  'Singapore Walking Tour',
  'Hong Kong Walking Tour',
  'Seoul Walking Tour',
  'Mumbai Walking Tour',
  'Istanbul Walking Tour',
  'Barcelona Walking Tour',
  'Amsterdam Walking Tour',
  'Stockholm Walking Tour',
];

/**
 * Sök efter videos på YouTube
 */
async function searchVideos(query, maxResults = 50) {
  console.log(`[DISCOVERY] Searching: "${query}"`);
  
  try {
    const response = await youtube.search.list({
      part: 'snippet',
      q: query,
      type: 'video',
      videoDefinition: 'high', // Endast HD+
      videoDuration: 'medium', // 4-20 minuter
      maxResults: maxResults,
      order: 'viewCount', // Sortera efter visningar
      publishedAfter: new Date(Date.now() - 365 * 24 * 60 * 60 * 1000).toISOString(), // Senaste året
    });
    
    return response.data.items.map(item => ({
      videoId: item.id.videoId,
      title: item.snippet.title,
      description: item.snippet.description,
      channelId: item.snippet.channelId,
      channelTitle: item.snippet.channelTitle,
      publishedAt: item.snippet.publishedAt,
      thumbnails: item.snippet.thumbnails,
      query: query,
    }));
  } catch (error) {
    console.error(`[DISCOVERY] Search failed for "${query}":`, error.message);
    return [];
  }
}

/**
 * Hämta detaljerad video-information
 */
async function getVideoDetails(videoIds) {
  console.log(`[DISCOVERY] Fetching details for ${videoIds.length} videos`);
  
  try {
    const response = await youtube.videos.list({
      part: 'contentDetails,statistics,status,recordingDetails',
      id: videoIds.join(','),
    });
    
    return response.data.items.map(item => ({
      videoId: item.id,
      duration: parseDuration(item.contentDetails.duration),
      dimension: item.contentDetails.dimension,
      definition: item.contentDetails.definition,
      caption: item.contentDetails.caption,
      licensedContent: item.contentDetails.licensedContent,
      projection: item.contentDetails.projection,
      viewCount: parseInt(item.statistics.viewCount || 0),
      likeCount: parseInt(item.statistics.likeCount || 0),
      commentCount: parseInt(item.statistics.commentCount || 0),
      privacyStatus: item.status.privacyStatus,
      license: item.status.license, // 'youtube' eller 'creativeCommon'
      embeddable: item.status.embeddable,
      publicStatsViewable: item.status.publicStatsViewable,
      location: item.recordingDetails?.location || null,
      recordingDate: item.recordingDetails?.recordingDate || null,
    }));
  } catch (error) {
    console.error('[DISCOVERY] Failed to fetch video details:', error.message);
    return [];
  }
}

/**
 * Parsa ISO 8601 duration (PT4M13S → 253 sekunder)
 */
function parseDuration(duration) {
  const match = duration.match(/PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?/);
  if (!match) return 0;
  
  const hours = parseInt(match[1] || 0);
  const minutes = parseInt(match[2] || 0);
  const seconds = parseInt(match[3] || 0);
  
  return hours * 3600 + minutes * 60 + seconds;
}

/**
 * Filtrera endast Creative Commons-videos
 */
function filterCreativeCommons(videos) {
  return videos.filter(video => video.license === 'creativeCommon');
}

/**
 * Extrahera stad från titel/beskrivning
 */
function extractCity(video) {
  const text = `${video.title} ${video.description}`.toLowerCase();
  
  const cities = [
    'bangkok', 'tokyo', 'london', 'paris', 'berlin', 'new york', 'dubai',
    'singapore', 'hong kong', 'seoul', 'mumbai', 'istanbul', 'barcelona',
    'amsterdam', 'stockholm', 'oslo', 'copenhagen', 'helsinki', 'vienna',
    'zurich', 'madrid', 'rome', 'prague', 'budapest', 'warsaw', 'moscow',
    'beijing', 'shanghai', 'sydney', 'melbourne', 'toronto', 'vancouver',
    'los angeles', 'chicago', 'miami', 'san francisco', 'boston',
  ];
  
  for (const city of cities) {
    if (text.includes(city)) {
      return city.replace(/\b\w/g, l => l.toUpperCase());
    }
  }
  
  return 'Unknown';
}

/**
 * Extrahera land från stad
 */
function extractCountry(city) {
  const cityCountryMap = {
    'Bangkok': 'Thailand',
    'Tokyo': 'Japan',
    'London': 'UK',
    'Paris': 'France',
    'Berlin': 'Germany',
    'New York': 'USA',
    'Dubai': 'UAE',
    'Singapore': 'Singapore',
    'Hong Kong': 'China',
    'Seoul': 'South Korea',
    'Mumbai': 'India',
    'Istanbul': 'Turkey',
    'Barcelona': 'Spain',
    'Amsterdam': 'Netherlands',
    'Stockholm': 'Sweden',
    'Oslo': 'Norway',
    'Copenhagen': 'Denmark',
    'Helsinki': 'Finland',
    'Vienna': 'Austria',
    'Zurich': 'Switzerland',
    'Madrid': 'Spain',
  };
  
  return cityCountryMap[city] || 'Unknown';
}

/**
 * Spara video till Redis-index
 */
async function indexVideo(video) {
  const key = `quixzoom:video:${video.videoId}`;
  
  await redis.hset(key, {
    videoId: video.videoId,
    title: video.title,
    description: video.description,
    channelId: video.channelId,
    channelTitle: video.channelTitle,
    publishedAt: video.publishedAt,
    duration: video.duration,
    definition: video.definition,
    viewCount: video.viewCount,
    likeCount: video.likeCount,
    license: video.license,
    city: video.city,
    country: video.country,
    location: JSON.stringify(video.location),
    query: video.query,
    indexedAt: new Date().toISOString(),
    status: 'discovered',
  });
  
  // Lägg till i stadsspecifik set
  await redis.sadd(`quixzoom:videos:city:${video.city.toLowerCase()}`, video.videoId);
  
  // Lägg till i landsspecifik set
  await redis.sadd(`quixzoom:videos:country:${video.country.toLowerCase()}`, video.videoId);
  
  // Lägg till i kö för nedladdning
  if (video.license === 'creativeCommon') {
    await redis.lpush('quixzoom:download:queue', JSON.stringify({
      videoId: video.videoId,
      title: video.title,
      city: video.city,
    }));
  }
}

/**
 * Huvudfunktion — kör discovery
 */
async function runDiscovery() {
  console.log('[DISCOVERY] Starting YouTube video discovery...');
  
  const allQueries = [...SEARCH_QUERIES, ...CITY_QUERIES];
  let totalDiscovered = 0;
  let totalCC = 0;
  
  for (const query of allQueries) {
    // Sök videos
    const searchResults = await searchVideos(query, 25);
    
    if (searchResults.length === 0) continue;
    
    // Hämta detaljer
    const videoIds = searchResults.map(r => r.videoId);
    const details = await getVideoDetails(videoIds);
    
    // Kombinera sökresultat med detaljer
    const videos = searchResults.map(search => {
      const detail = details.find(d => d.videoId === search.videoId);
      return { ...search, ...detail };
    });
    
    // Filtrera Creative Commons
    const ccVideos = filterCreativeCommons(videos);
    
    // Extrahera stad och land
    for (const video of ccVideos) {
      video.city = extractCity(video);
      video.country = extractCountry(video.city);
      
      // Indexera
      await indexVideo(video);
      
      console.log(`[DISCOVERY] Indexed: ${video.title} (${video.city}, ${video.country}) [CC]`);
    }
    
    totalDiscovered += videos.length;
    totalCC += ccVideos.length;
    
    // Rate limiting — vänta mellan anrop
    await new Promise(resolve => setTimeout(resolve, 1000));
  }
  
  console.log(`[DISCOVERY] Complete: ${totalDiscovered} videos discovered, ${totalCC} Creative Commons`);
  
  // Spara statistik
  await redis.incrby('quixzoom:stats:discovery:total', totalDiscovered);
  await redis.incrby('quixzoom:stats:discovery:cc', totalCC);
}

/**
 * Hämta indexerade videos för en stad
 */
async function getVideosByCity(city) {
  const videoIds = await redis.smembers(`quixzoom:videos:city:${city.toLowerCase()}`);
  
  const videos = [];
  for (const videoId of videoIds) {
    const video = await redis.hgetall(`quixzoom:video:${videoId}`);
    if (Object.keys(video).length > 0) {
      videos.push(video);
    }
  }
  
  return videos;
}

/**
 * Hämta statistik
 */
async function getStats() {
  const total = await redis.get('quixzoom:stats:discovery:total') || 0;
  const cc = await redis.get('quixzoom:stats:discovery:cc') || 0;
  const queue = await redis.llen('quixzoom:download:queue');
  
  return { total: parseInt(total), cc: parseInt(cc), downloadQueue: queue };
}

// Kör discovery om filen körs direkt
if (require.main === module) {
  runDiscovery()
    .then(() => process.exit(0))
    .catch(err => {
      console.error(err);
      process.exit(1);
    });
}

module.exports = {
  runDiscovery,
  searchVideos,
  getVideoDetails,
  filterCreativeCommons,
  extractCity,
  extractCountry,
  getVideosByCity,
  getStats,
};