455 lines
16 KiB
JavaScript
455 lines
16 KiB
JavaScript
import { XMLParser } from 'fast-xml-parser';
|
|
import OpenAI from 'openai';
|
|
import * as cheerio from 'cheerio';
|
|
import { d as downloadAndStoreImage } from './images_DSuHPXn1.mjs';
|
|
import { p as publishPost } from './publisher_Dj_UvF8J.mjs';
|
|
import { r as readCms, s as slugify, w as writeCms } from './cms_CRTVymfB.mjs';
|
|
import puppeteer from 'puppeteer-core';
|
|
import { existsSync } from 'node:fs';
|
|
|
|
let browserInstance = null;
|
|
const getChromePath = () => {
|
|
const platform = process.platform;
|
|
if (platform === "darwin") {
|
|
const paths = [
|
|
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
|
"/Applications/Chromium.app/Contents/MacOS/Chromium",
|
|
"/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"
|
|
];
|
|
for (const p of paths) {
|
|
if (existsSync(p)) return p;
|
|
}
|
|
}
|
|
if (platform === "linux") {
|
|
const paths = [
|
|
"/usr/bin/google-chrome",
|
|
"/usr/bin/google-chrome-stable",
|
|
"/usr/bin/chromium",
|
|
"/usr/bin/chromium-browser",
|
|
"/snap/bin/chromium"
|
|
];
|
|
for (const p of paths) {
|
|
if (existsSync(p)) return p;
|
|
}
|
|
}
|
|
if (platform === "win32") {
|
|
const paths = [
|
|
"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
|
|
"C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe"
|
|
];
|
|
for (const p of paths) {
|
|
if (existsSync(p)) return p;
|
|
}
|
|
}
|
|
return "";
|
|
};
|
|
const getBrowser$1 = async () => {
|
|
if (browserInstance && browserInstance.connected) {
|
|
return browserInstance;
|
|
}
|
|
const chromePath = process.env.PUPPETEER_EXECUTABLE_PATH || getChromePath();
|
|
if (!chromePath) {
|
|
throw new Error("Chrome/Chromium not found. Install Chrome or set PUPPETEER_EXECUTABLE_PATH.");
|
|
}
|
|
browserInstance = await puppeteer.launch({
|
|
executablePath: chromePath,
|
|
headless: true,
|
|
args: [
|
|
"--no-sandbox",
|
|
"--disable-setuid-sandbox",
|
|
"--disable-dev-shm-usage",
|
|
"--disable-gpu",
|
|
"--window-size=1920,1080"
|
|
]
|
|
});
|
|
return browserInstance;
|
|
};
|
|
const closeBrowser = async () => {
|
|
if (browserInstance) {
|
|
await browserInstance.close();
|
|
browserInstance = null;
|
|
}
|
|
};
|
|
const scrapeArticle = async (url) => {
|
|
const browser = await getBrowser$1();
|
|
const page = await browser.newPage();
|
|
try {
|
|
await page.setViewport({ width: 1920, height: 1080 });
|
|
await page.setUserAgent(
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
|
|
);
|
|
await page.setRequestInterception(true);
|
|
page.on("request", (req) => {
|
|
const type = req.resourceType();
|
|
if (["font", "media"].includes(type)) {
|
|
req.abort();
|
|
} else {
|
|
req.continue();
|
|
}
|
|
});
|
|
await page.goto(url, {
|
|
waitUntil: "domcontentloaded",
|
|
timeout: 6e4
|
|
});
|
|
await new Promise((r) => setTimeout(r, 2e3));
|
|
const data = await page.evaluate(() => {
|
|
const title = document.querySelector('meta[property="og:title"]')?.getAttribute("content") || document.querySelector("h1")?.textContent?.trim() || document.title;
|
|
const description = document.querySelector('meta[property="og:description"]')?.getAttribute("content") || document.querySelector('meta[name="description"]')?.getAttribute("content") || "";
|
|
const ogImage = document.querySelector('meta[property="og:image"]')?.getAttribute("content") || "";
|
|
const allImages = [];
|
|
const imageBlacklist = ["logo", "icon", "avatar", "counter", "yadro", "pixel", "track", "1x1", "badge"];
|
|
document.querySelectorAll("img").forEach((img) => {
|
|
const src = img.getAttribute("src") || img.getAttribute("data-src") || "";
|
|
if (!src) return;
|
|
const lowerSrc = src.toLowerCase();
|
|
if (imageBlacklist.some((b) => lowerSrc.includes(b))) return;
|
|
const width = parseInt(img.getAttribute("width") || "0");
|
|
const height = parseInt(img.getAttribute("height") || "0");
|
|
if (width > 0 && width < 50 || height > 0 && height < 50) return;
|
|
try {
|
|
allImages.push(new URL(src, window.location.origin).href);
|
|
} catch {
|
|
}
|
|
});
|
|
const contentContainer = document.querySelector('[class*="ContentPageContainer_content"]') || document.querySelector("article") || document.querySelector("[data-content]");
|
|
const articleImage = contentContainer?.querySelector("img")?.getAttribute("src") || "";
|
|
const imageUrl = ogImage || articleImage || "";
|
|
const paragraphs = [];
|
|
const footerTexts = /* @__PURE__ */ new Set();
|
|
document.querySelectorAll('[class*="Legal"], [class*="legal"], [class*="footer"], [class*="Footer"], [class*="Copyright"], [class*="copyright"], [class*="sidebar"], [class*="Sidebar"]').forEach((el) => {
|
|
el.querySelectorAll("p, li").forEach((p) => {
|
|
const t = p.textContent?.trim().replace(/\s+/g, " ") || "";
|
|
if (t) footerTexts.add(t);
|
|
});
|
|
});
|
|
const selectors = [
|
|
'[class*="ContentPageContainer_content"] p',
|
|
'[class*="article"] p',
|
|
'[class*="Article"] p',
|
|
'[class*="news-text"] p',
|
|
'[class*="detail"] p',
|
|
'[class*="topic"] p',
|
|
"article p",
|
|
"[data-content] p",
|
|
".text-content p",
|
|
".article-body p"
|
|
];
|
|
for (const selector of selectors) {
|
|
document.querySelectorAll(selector).forEach((el) => {
|
|
const text = el.textContent?.trim().replace(/\s+/g, " ") || "";
|
|
if (text.length > 20 && !paragraphs.includes(text) && !footerTexts.has(text)) {
|
|
paragraphs.push(text);
|
|
}
|
|
});
|
|
if (paragraphs.length > 0) break;
|
|
}
|
|
if (paragraphs.length < 2) {
|
|
document.querySelectorAll("main p, .content p, .post-content p").forEach((el) => {
|
|
const text = el.textContent?.trim().replace(/\s+/g, " ") || "";
|
|
if (text.length > 30 && !paragraphs.includes(text) && !footerTexts.has(text)) {
|
|
paragraphs.push(text);
|
|
}
|
|
});
|
|
}
|
|
if (paragraphs.length < 2) {
|
|
document.querySelectorAll("p").forEach((el) => {
|
|
const text = el.textContent?.trim().replace(/\s+/g, " ") || "";
|
|
if (text.length > 60 && !paragraphs.includes(text) && !footerTexts.has(text)) {
|
|
paragraphs.push(text);
|
|
}
|
|
});
|
|
}
|
|
return {
|
|
title: title?.trim() || "",
|
|
description: description?.trim() || "",
|
|
imageUrl,
|
|
text: paragraphs.slice(0, 14).join("\n\n"),
|
|
allImages
|
|
};
|
|
});
|
|
const resolvedImages = data.allImages.map((img) => {
|
|
try {
|
|
return new URL(img, url).href;
|
|
} catch {
|
|
return img;
|
|
}
|
|
});
|
|
return {
|
|
...data,
|
|
allImages: resolvedImages,
|
|
imageUrl: data.imageUrl ? (() => {
|
|
try {
|
|
return new URL(data.imageUrl, url).href;
|
|
} catch {
|
|
return data.imageUrl;
|
|
}
|
|
})() : ""
|
|
};
|
|
} finally {
|
|
await page.close();
|
|
}
|
|
};
|
|
|
|
const rssUrl = "https://tass.ru/rss/v2.xml";
|
|
const getOpenAIClient = () => new OpenAI({
|
|
apiKey: "sk-C9kUNuxFPA6Jvpi2stzikxzZMJzPuYy5TIoJEJiVR89oeiD6",
|
|
baseURL: "https://api.gapgpt.app/v1"
|
|
});
|
|
const getModel = () => "gpt-5-nano";
|
|
const fetchText = async (url) => {
|
|
try {
|
|
const response = await fetch(url, {
|
|
headers: {
|
|
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
|
|
accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
"accept-language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
|
|
referer: "https://tass.ru/"
|
|
},
|
|
signal: AbortSignal.timeout(8e3)
|
|
});
|
|
if (response.ok) {
|
|
return response.text();
|
|
}
|
|
} catch {
|
|
}
|
|
const browser = await getBrowser();
|
|
const page = await browser.newPage();
|
|
try {
|
|
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 3e4 });
|
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
const content = await page.evaluate(() => {
|
|
const pre = document.querySelector("pre");
|
|
if (pre) return pre.textContent || "";
|
|
return document.body.innerText || document.body.textContent || "";
|
|
});
|
|
return content;
|
|
} finally {
|
|
await page.close();
|
|
}
|
|
};
|
|
const getTassFeedLinks = async (limit = 50) => {
|
|
const xml = await fetchText(rssUrl);
|
|
const parser = new XMLParser({ ignoreAttributes: false });
|
|
const parsed = parser.parse(xml);
|
|
const rawItems = parsed?.rss?.channel?.item ?? [];
|
|
const items = Array.isArray(rawItems) ? rawItems : [rawItems];
|
|
return items.slice(0, limit).map((item) => item.link).filter(Boolean);
|
|
};
|
|
const getItems = async (limit) => {
|
|
const xml = await fetchText(rssUrl);
|
|
const parser = new XMLParser({ ignoreAttributes: false });
|
|
const parsed = parser.parse(xml);
|
|
const rawItems = parsed?.rss?.channel?.item ?? [];
|
|
const items = Array.isArray(rawItems) ? rawItems : [rawItems];
|
|
return items.slice(0, Math.max(limit * 10, 30));
|
|
};
|
|
const stripHtml = (value = "") => {
|
|
if (!value) return "";
|
|
const $ = cheerio.load(value);
|
|
return $.text().replace(/\s+/g, " ").trim();
|
|
};
|
|
const getRssImage = (item) => {
|
|
const candidates = [
|
|
item.enclosure?.["@_type"]?.startsWith("image/") ? item.enclosure?.["@_url"] : "",
|
|
item["media:content"]?.["@_type"]?.startsWith("image/") ? item["media:content"]?.["@_url"] : "",
|
|
item["media:thumbnail"]?.["@_url"]
|
|
].filter(Boolean);
|
|
return candidates.find((candidate) => candidate.startsWith("http")) ?? "";
|
|
};
|
|
const sortImageItemsFirst = (items) => [...items].sort((first, second) => {
|
|
const firstHasImage = getRssImage(first) ? 1 : 0;
|
|
const secondHasImage = getRssImage(second) ? 1 : 0;
|
|
return secondHasImage - firstHasImage;
|
|
});
|
|
const getFallbackArticle = (item) => {
|
|
const description = stripHtml(item.description);
|
|
return {
|
|
title: item.title ?? "",
|
|
description,
|
|
imageUrl: getRssImage(item),
|
|
text: description,
|
|
usedFallback: true
|
|
};
|
|
};
|
|
const toAbsoluteUrl = (value, baseUrl) => {
|
|
try {
|
|
return new URL(value, baseUrl).toString();
|
|
} catch {
|
|
return "";
|
|
}
|
|
};
|
|
const extractArticleCheerio = async (url) => {
|
|
const html = await fetchText(url);
|
|
const $ = cheerio.load(html);
|
|
const title = $('meta[property="og:title"]').attr("content") || $("h1").first().text().replace(/\s+/g, " ").trim();
|
|
const description = $('meta[property="og:description"]').attr("content") || "";
|
|
const imageCandidates = [
|
|
$('meta[property="og:image"]').attr("content"),
|
|
$("article img").first().attr("src"),
|
|
$("img").first().attr("src")
|
|
].filter(Boolean);
|
|
const imageUrl = imageCandidates.map((c) => toAbsoluteUrl(c, url)).find(Boolean) ?? "";
|
|
const paragraphs = [];
|
|
$("article p, [data-content] p, .text-content p, p").each((_, el) => {
|
|
const pText = $(el).text().replace(/\s+/g, " ").trim();
|
|
if (pText.length > 80) {
|
|
paragraphs.push(pText);
|
|
}
|
|
});
|
|
return {
|
|
title,
|
|
description,
|
|
imageUrl,
|
|
text: paragraphs.slice(0, 14).join("\n\n")
|
|
};
|
|
};
|
|
const extractArticle = async (url) => {
|
|
try {
|
|
const scraped = await scrapeArticle(url);
|
|
if (scraped.text || scraped.title) {
|
|
return {
|
|
title: scraped.title,
|
|
description: scraped.description,
|
|
imageUrl: scraped.imageUrl,
|
|
text: scraped.text,
|
|
usedFallback: false
|
|
};
|
|
}
|
|
} catch {
|
|
}
|
|
const fallback = await extractArticleCheerio(url);
|
|
return {
|
|
...fallback,
|
|
usedFallback: true
|
|
};
|
|
};
|
|
const translateArticle = async (article) => {
|
|
const client = getOpenAIClient();
|
|
if (!client.apiKey) {
|
|
throw new Error("GAPGPT_API_KEY or OPENAI_API_KEY is not configured");
|
|
}
|
|
const sourceText = article.text || article.description || article.title;
|
|
if (!sourceText) {
|
|
throw new Error("RSS item has no usable text");
|
|
}
|
|
const completion = await client.chat.completions.create({
|
|
model: getModel(),
|
|
messages: [
|
|
{
|
|
role: "system",
|
|
content: "تو سردبیر فارسی یک سایت خبری هستی. متن روسی/انگلیسی خبر را به فارسی روان و خبری تبدیل کن. خروجی فقط JSON معتبر باشد."
|
|
},
|
|
{
|
|
role: "user",
|
|
content: JSON.stringify({
|
|
instruction: "ترجمه و بازنویسی خبری فارسی بساز. دسته یکی از اقتصاد، سیاست، انرژی، فناوری، بینالملل باشد. body آرایه پاراگرافها باشد.",
|
|
source: { ...article, text: sourceText },
|
|
schema: {
|
|
title: "string",
|
|
excerpt: "string",
|
|
category: "string",
|
|
readTime: "string",
|
|
body: ["string"]
|
|
}
|
|
})
|
|
}
|
|
],
|
|
response_format: { type: "json_object" },
|
|
temperature: 0.3
|
|
});
|
|
const content = completion.choices[0]?.message?.content ?? "{}";
|
|
const jsonText = content.match(/\{[\s\S]*\}/)?.[0] ?? content;
|
|
const parsed = JSON.parse(jsonText);
|
|
return {
|
|
title: parsed.title?.trim() || article.title,
|
|
excerpt: parsed.excerpt?.trim() || article.description,
|
|
category: parsed.category?.trim() || "بینالملل",
|
|
readTime: parsed.readTime?.trim() || "۳ دقیقه",
|
|
body: Array.isArray(parsed.body) ? parsed.body.map(String).filter(Boolean) : [article.text]
|
|
};
|
|
};
|
|
const runTassAgent = async (limit = 3) => {
|
|
const cms = await readCms();
|
|
const processedLinks = new Set(cms.settings.agents?.tass?.processedLinks ?? []);
|
|
const items = sortImageItemsFirst(await getItems(limit));
|
|
const created = [];
|
|
const failed = [];
|
|
const skipped = [];
|
|
for (const item of items) {
|
|
if (created.length >= limit) {
|
|
break;
|
|
}
|
|
const link = item.link;
|
|
if (!link) {
|
|
continue;
|
|
}
|
|
if (processedLinks.has(link)) {
|
|
skipped.push(link);
|
|
continue;
|
|
}
|
|
try {
|
|
let article = getFallbackArticle(item);
|
|
let extractionWarning = "";
|
|
try {
|
|
const extractedArticle = await extractArticle(link);
|
|
article = {
|
|
...article,
|
|
...extractedArticle,
|
|
title: extractedArticle.title || article.title,
|
|
description: extractedArticle.description || article.description,
|
|
imageUrl: extractedArticle.imageUrl || article.imageUrl,
|
|
text: extractedArticle.text || article.text
|
|
};
|
|
} catch (error) {
|
|
extractionWarning = error instanceof Error ? error.message : "Article extraction failed";
|
|
}
|
|
const translated = await translateArticle({
|
|
title: article.title || item.title || "",
|
|
description: article.description || item.description || "",
|
|
text: article.text || item.description || ""
|
|
});
|
|
const storedImage = article.imageUrl ? await downloadAndStoreImage(article.imageUrl) : null;
|
|
const body = [...translated.body];
|
|
if (storedImage?.image) {
|
|
body.splice(1, 0, `[image:${storedImage.image}|${translated.title}]`);
|
|
}
|
|
const post = await publishPost({
|
|
title: translated.title,
|
|
slug: slugify(`${translated.title}-${Date.now()}`),
|
|
category: translated.category,
|
|
readTime: translated.readTime,
|
|
excerpt: translated.excerpt,
|
|
body,
|
|
status: "pending",
|
|
image: storedImage?.image,
|
|
thumbnail: storedImage?.thumbnail,
|
|
sourceUrl: link
|
|
});
|
|
processedLinks.add(link);
|
|
created.push({
|
|
slug: post.slug,
|
|
title: post.title,
|
|
source: link,
|
|
sourceImage: article.imageUrl || void 0,
|
|
image: storedImage?.image,
|
|
thumbnail: storedImage?.thumbnail,
|
|
warning: extractionWarning || void 0
|
|
});
|
|
} catch (error) {
|
|
failed.push({ source: link, error: error instanceof Error ? error.message : "Unknown error" });
|
|
}
|
|
}
|
|
const latestCms = await readCms();
|
|
latestCms.settings.agents = {
|
|
...latestCms.settings.agents,
|
|
tass: {
|
|
processedLinks: Array.from(processedLinks)
|
|
}
|
|
};
|
|
await writeCms(latestCms);
|
|
await closeBrowser();
|
|
return { created, failed, skipped };
|
|
};
|
|
|
|
export { getTassFeedLinks as g, runTassAgent as r };
|