import { XMLParser } from 'fast-xml-parser'; import OpenAI from 'openai'; import * as cheerio from 'cheerio'; import { d as downloadAndStoreImage } from './images_DSuHPXn1.mjs'; import { p as publishPost } from './publisher_Dj_UvF8J.mjs'; import { r as readCms, s as slugify, w as writeCms } from './cms_CRTVymfB.mjs'; import puppeteer from 'puppeteer-core'; import { existsSync } from 'node:fs'; let browserInstance = null; const getChromePath = () => { const platform = process.platform; if (platform === "darwin") { const paths = [ "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", "/Applications/Chromium.app/Contents/MacOS/Chromium", "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser" ]; for (const p of paths) { if (existsSync(p)) return p; } } if (platform === "linux") { const paths = [ "/usr/bin/google-chrome", "/usr/bin/google-chrome-stable", "/usr/bin/chromium", "/usr/bin/chromium-browser", "/snap/bin/chromium" ]; for (const p of paths) { if (existsSync(p)) return p; } } if (platform === "win32") { const paths = [ "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe", "C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe" ]; for (const p of paths) { if (existsSync(p)) return p; } } return ""; }; const getBrowser$1 = async () => { if (browserInstance && browserInstance.connected) { return browserInstance; } const chromePath = process.env.PUPPETEER_EXECUTABLE_PATH || getChromePath(); if (!chromePath) { throw new Error("Chrome/Chromium not found. Install Chrome or set PUPPETEER_EXECUTABLE_PATH."); } browserInstance = await puppeteer.launch({ executablePath: chromePath, headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", "--disable-gpu", "--window-size=1920,1080" ] }); return browserInstance; }; const closeBrowser = async () => { if (browserInstance) { await browserInstance.close(); browserInstance = null; } }; const scrapeArticle = async (url) => { const browser = await getBrowser$1(); const page = await browser.newPage(); try { await page.setViewport({ width: 1920, height: 1080 }); await page.setUserAgent( "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36" ); await page.setRequestInterception(true); page.on("request", (req) => { const type = req.resourceType(); if (["font", "media"].includes(type)) { req.abort(); } else { req.continue(); } }); await page.goto(url, { waitUntil: "domcontentloaded", timeout: 6e4 }); await new Promise((r) => setTimeout(r, 2e3)); const data = await page.evaluate(() => { const title = document.querySelector('meta[property="og:title"]')?.getAttribute("content") || document.querySelector("h1")?.textContent?.trim() || document.title; const description = document.querySelector('meta[property="og:description"]')?.getAttribute("content") || document.querySelector('meta[name="description"]')?.getAttribute("content") || ""; const ogImage = document.querySelector('meta[property="og:image"]')?.getAttribute("content") || ""; const allImages = []; const imageBlacklist = ["logo", "icon", "avatar", "counter", "yadro", "pixel", "track", "1x1", "badge"]; document.querySelectorAll("img").forEach((img) => { const src = img.getAttribute("src") || img.getAttribute("data-src") || ""; if (!src) return; const lowerSrc = src.toLowerCase(); if (imageBlacklist.some((b) => lowerSrc.includes(b))) return; const width = parseInt(img.getAttribute("width") || "0"); const height = parseInt(img.getAttribute("height") || "0"); if (width > 0 && width < 50 || height > 0 && height < 50) return; try { allImages.push(new URL(src, window.location.origin).href); } catch { } }); const contentContainer = document.querySelector('[class*="ContentPageContainer_content"]') || document.querySelector("article") || document.querySelector("[data-content]"); const articleImage = contentContainer?.querySelector("img")?.getAttribute("src") || ""; const imageUrl = ogImage || articleImage || ""; const paragraphs = []; const footerTexts = /* @__PURE__ */ new Set(); document.querySelectorAll('[class*="Legal"], [class*="legal"], [class*="footer"], [class*="Footer"], [class*="Copyright"], [class*="copyright"], [class*="sidebar"], [class*="Sidebar"]').forEach((el) => { el.querySelectorAll("p, li").forEach((p) => { const t = p.textContent?.trim().replace(/\s+/g, " ") || ""; if (t) footerTexts.add(t); }); }); const selectors = [ '[class*="ContentPageContainer_content"] p', '[class*="article"] p', '[class*="Article"] p', '[class*="news-text"] p', '[class*="detail"] p', '[class*="topic"] p', "article p", "[data-content] p", ".text-content p", ".article-body p" ]; for (const selector of selectors) { document.querySelectorAll(selector).forEach((el) => { const text = el.textContent?.trim().replace(/\s+/g, " ") || ""; if (text.length > 20 && !paragraphs.includes(text) && !footerTexts.has(text)) { paragraphs.push(text); } }); if (paragraphs.length > 0) break; } if (paragraphs.length < 2) { document.querySelectorAll("main p, .content p, .post-content p").forEach((el) => { const text = el.textContent?.trim().replace(/\s+/g, " ") || ""; if (text.length > 30 && !paragraphs.includes(text) && !footerTexts.has(text)) { paragraphs.push(text); } }); } if (paragraphs.length < 2) { document.querySelectorAll("p").forEach((el) => { const text = el.textContent?.trim().replace(/\s+/g, " ") || ""; if (text.length > 60 && !paragraphs.includes(text) && !footerTexts.has(text)) { paragraphs.push(text); } }); } return { title: title?.trim() || "", description: description?.trim() || "", imageUrl, text: paragraphs.slice(0, 14).join("\n\n"), allImages }; }); const resolvedImages = data.allImages.map((img) => { try { return new URL(img, url).href; } catch { return img; } }); return { ...data, allImages: resolvedImages, imageUrl: data.imageUrl ? (() => { try { return new URL(data.imageUrl, url).href; } catch { return data.imageUrl; } })() : "" }; } finally { await page.close(); } }; const rssUrl = "https://tass.ru/rss/v2.xml"; const getOpenAIClient = () => new OpenAI({ apiKey: "sk-C9kUNuxFPA6Jvpi2stzikxzZMJzPuYy5TIoJEJiVR89oeiD6", baseURL: "https://api.gapgpt.app/v1" }); const getModel = () => "gpt-5-nano"; const fetchText = async (url) => { try { const response = await fetch(url, { headers: { "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "accept-language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7", referer: "https://tass.ru/" }, signal: AbortSignal.timeout(8e3) }); if (response.ok) { return response.text(); } } catch { } const browser = await getBrowser(); const page = await browser.newPage(); try { await page.goto(url, { waitUntil: "domcontentloaded", timeout: 3e4 }); await new Promise((r) => setTimeout(r, 1e3)); const content = await page.evaluate(() => { const pre = document.querySelector("pre"); if (pre) return pre.textContent || ""; return document.body.innerText || document.body.textContent || ""; }); return content; } finally { await page.close(); } }; const getTassFeedLinks = async (limit = 50) => { const xml = await fetchText(rssUrl); const parser = new XMLParser({ ignoreAttributes: false }); const parsed = parser.parse(xml); const rawItems = parsed?.rss?.channel?.item ?? []; const items = Array.isArray(rawItems) ? rawItems : [rawItems]; return items.slice(0, limit).map((item) => item.link).filter(Boolean); }; const getItems = async (limit) => { const xml = await fetchText(rssUrl); const parser = new XMLParser({ ignoreAttributes: false }); const parsed = parser.parse(xml); const rawItems = parsed?.rss?.channel?.item ?? []; const items = Array.isArray(rawItems) ? rawItems : [rawItems]; return items.slice(0, Math.max(limit * 10, 30)); }; const stripHtml = (value = "") => { if (!value) return ""; const $ = cheerio.load(value); return $.text().replace(/\s+/g, " ").trim(); }; const getRssImage = (item) => { const candidates = [ item.enclosure?.["@_type"]?.startsWith("image/") ? item.enclosure?.["@_url"] : "", item["media:content"]?.["@_type"]?.startsWith("image/") ? item["media:content"]?.["@_url"] : "", item["media:thumbnail"]?.["@_url"] ].filter(Boolean); return candidates.find((candidate) => candidate.startsWith("http")) ?? ""; }; const sortImageItemsFirst = (items) => [...items].sort((first, second) => { const firstHasImage = getRssImage(first) ? 1 : 0; const secondHasImage = getRssImage(second) ? 1 : 0; return secondHasImage - firstHasImage; }); const getFallbackArticle = (item) => { const description = stripHtml(item.description); return { title: item.title ?? "", description, imageUrl: getRssImage(item), text: description, usedFallback: true }; }; const toAbsoluteUrl = (value, baseUrl) => { try { return new URL(value, baseUrl).toString(); } catch { return ""; } }; const extractArticleCheerio = async (url) => { const html = await fetchText(url); const $ = cheerio.load(html); const title = $('meta[property="og:title"]').attr("content") || $("h1").first().text().replace(/\s+/g, " ").trim(); const description = $('meta[property="og:description"]').attr("content") || ""; const imageCandidates = [ $('meta[property="og:image"]').attr("content"), $("article img").first().attr("src"), $("img").first().attr("src") ].filter(Boolean); const imageUrl = imageCandidates.map((c) => toAbsoluteUrl(c, url)).find(Boolean) ?? ""; const paragraphs = []; $("article p, [data-content] p, .text-content p, p").each((_, el) => { const pText = $(el).text().replace(/\s+/g, " ").trim(); if (pText.length > 80) { paragraphs.push(pText); } }); return { title, description, imageUrl, text: paragraphs.slice(0, 14).join("\n\n") }; }; const extractArticle = async (url) => { try { const scraped = await scrapeArticle(url); if (scraped.text || scraped.title) { return { title: scraped.title, description: scraped.description, imageUrl: scraped.imageUrl, text: scraped.text, usedFallback: false }; } } catch { } const fallback = await extractArticleCheerio(url); return { ...fallback, usedFallback: true }; }; const translateArticle = async (article) => { const client = getOpenAIClient(); if (!client.apiKey) { throw new Error("GAPGPT_API_KEY or OPENAI_API_KEY is not configured"); } const sourceText = article.text || article.description || article.title; if (!sourceText) { throw new Error("RSS item has no usable text"); } const completion = await client.chat.completions.create({ model: getModel(), messages: [ { role: "system", content: "تو سردبیر فارسی یک سایت خبری هستی. متن روسی/انگلیسی خبر را به فارسی روان و خبری تبدیل کن. خروجی فقط JSON معتبر باشد." }, { role: "user", content: JSON.stringify({ instruction: "ترجمه و بازنویسی خبری فارسی بساز. دسته یکی از اقتصاد، سیاست، انرژی، فناوری، بین‌الملل باشد. body آرایه پاراگراف‌ها باشد.", source: { ...article, text: sourceText }, schema: { title: "string", excerpt: "string", category: "string", readTime: "string", body: ["string"] } }) } ], response_format: { type: "json_object" }, temperature: 0.3 }); const content = completion.choices[0]?.message?.content ?? "{}"; const jsonText = content.match(/\{[\s\S]*\}/)?.[0] ?? content; const parsed = JSON.parse(jsonText); return { title: parsed.title?.trim() || article.title, excerpt: parsed.excerpt?.trim() || article.description, category: parsed.category?.trim() || "بین‌الملل", readTime: parsed.readTime?.trim() || "۳ دقیقه", body: Array.isArray(parsed.body) ? parsed.body.map(String).filter(Boolean) : [article.text] }; }; const runTassAgent = async (limit = 3) => { const cms = await readCms(); const processedLinks = new Set(cms.settings.agents?.tass?.processedLinks ?? []); const items = sortImageItemsFirst(await getItems(limit)); const created = []; const failed = []; const skipped = []; for (const item of items) { if (created.length >= limit) { break; } const link = item.link; if (!link) { continue; } if (processedLinks.has(link)) { skipped.push(link); continue; } try { let article = getFallbackArticle(item); let extractionWarning = ""; try { const extractedArticle = await extractArticle(link); article = { ...article, ...extractedArticle, title: extractedArticle.title || article.title, description: extractedArticle.description || article.description, imageUrl: extractedArticle.imageUrl || article.imageUrl, text: extractedArticle.text || article.text }; } catch (error) { extractionWarning = error instanceof Error ? error.message : "Article extraction failed"; } const translated = await translateArticle({ title: article.title || item.title || "", description: article.description || item.description || "", text: article.text || item.description || "" }); const storedImage = article.imageUrl ? await downloadAndStoreImage(article.imageUrl) : null; const body = [...translated.body]; if (storedImage?.image) { body.splice(1, 0, `[image:${storedImage.image}|${translated.title}]`); } const post = await publishPost({ title: translated.title, slug: slugify(`${translated.title}-${Date.now()}`), category: translated.category, readTime: translated.readTime, excerpt: translated.excerpt, body, status: "pending", image: storedImage?.image, thumbnail: storedImage?.thumbnail, sourceUrl: link }); processedLinks.add(link); created.push({ slug: post.slug, title: post.title, source: link, sourceImage: article.imageUrl || void 0, image: storedImage?.image, thumbnail: storedImage?.thumbnail, warning: extractionWarning || void 0 }); } catch (error) { failed.push({ source: link, error: error instanceof Error ? error.message : "Unknown error" }); } } const latestCms = await readCms(); latestCms.settings.agents = { ...latestCms.settings.agents, tass: { processedLinks: Array.from(processedLinks) } }; await writeCms(latestCms); await closeBrowser(); return { created, failed, skipped }; }; export { getTassFeedLinks as g, runTassAgent as r };