Miralinks: массовая проверка доноров с ценами и ссылками

(async () => {
  /*
    MIRALINKS BULK CHECKER v2
    Проверяет список доменов и выгружает:
    - наличие в каталоге
    - Ground.id
    - ссылку на карточку по Ground.id → /grounds/view/<ID>
    - цену размещения
    - цену написания
    - статус доступности
    - трафик
    - количество статей
    - скорость/последнее размещение
    - CF / TF / Ahrefs DR
    - число страниц в индексе Яндекса
    - category_ids


    Запускать в авторизованном Miralinks на странице "Каталог площадок".
  */


  const INPUT = `
example.com
example2.ru
example3.ru
`;


  const REQUESTED_DOMAINS = INPUT
    .split(/\r?\n|[,;\t ]+/)
    .map(x => x.trim())
    .filter(Boolean);


  const DISPLAY_LENGTH = 50;
  const PAGE_DELAY_MS = 2200;
  const MAX_RETRIES = 3;


  window.__ML_STOP__ = false;


  const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));


  function normalizeHost(value) {
    let s = String(value ?? '').trim().toLowerCase();
    if (!s) return '';


    s = s.replace(/\\\./g, '.');


    if (!/^https?:\/\//i.test(s)) {
      s = 'http://' + s;
    }


    try {
      return new URL(s).hostname
        .toLowerCase()
        .replace(/^www\./, '')
        .replace(/\.$/, '');
    } catch {
      return String(value ?? '')
        .trim()
        .toLowerCase()
        .replace(/\\\./g, '.')
        .replace(/^https?:\/\//i, '')
        .replace(/^www\./i, '')
        .split('/')[0]
        .split('?')[0]
        .split('#')[0]
        .replace(/\.$/, '');
    }
  }


  const requested = [...new Set(
    REQUESTED_DOMAINS.map(normalizeHost).filter(Boolean)
  )];


  const wanted = new Set(requested);
  const found = new Map();


  let requestsMade = 0;
  let catalogRowsSeen = 0;
  let visibleUrlsSeen = 0;


  function getSearchData() {
    try {
      return ML.Storages.getUsersRemoteStorage().__STORE__.lastCatalogFilter || {};
    } catch {
      throw new Error(
        'Не удалось прочитать фильтр Miralinks. Запусти скрипт прямо в "Каталоге площадок".'
      );
    }
  }


  function addSortingFields(form) {
    try {
      for (const key in (window.dataTables || {})) {
        if (!key.startsWith('Catalog_')) continue;


        const settings = window.dataTables[key]?.fnSettings?.();
        const sorting = settings?.aaSorting || [];


        form.append('iSortingCols', sorting.length || 1);


        sorting.forEach((item, i) => {
          form.append(`bSortable_${i}`, 'true');
          form.append(`iSortCol_${i}`, item[0]);
          form.append(`sSortDir_${i}`, item[1]);
        });


        return;
      }
    } catch (e) {
      console.warn('Не удалось прочитать сортировку таблицы. Продолжаю.', e);
    }


    form.append('iSortingCols', '0');
  }


  function generateBody(params = {}) {
    const form = new FormData();


    Object.entries(params).forEach(([key, value]) => {
      form.append(key, String(value));
    });


    form.append('tsDataTableType', 'with_checking');
    form.append('tsDataTableConfigType', 'dataTable.Catalog');
    form.append('iDisplayLength', String(DISPLAY_LENGTH));
    form.append('searchData', JSON.stringify(getSearchData()));


    addSortingFields(form);


    return form;
  }


  async function fetchCatalogPage(params) {
    let lastError;


    for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
      try {
        if (window.__ML_STOP__) {
          throw new Error('Проверка остановлена вручную.');
        }


        requestsMade++;


        const response = await fetch(
          `${location.origin}/ajaxPort/loadDataTableDataCatalog`,
          {
            method: 'POST',
            credentials: 'include',
            headers: {
              'x-requested-with': 'XMLHttpRequest'
            },
            body: generateBody(params)
          }
        );


        if (!response.ok) {
          throw new Error(`HTTP ${response.status}`);
        }


        const data = await response.json();


        if (!Array.isArray(data?.aaData)) {
          throw new Error('Miralinks вернул неожиданный ответ.');
        }


        return data;
      } catch (e) {
        lastError = e;
        console.warn(`Ошибка запроса, попытка ${attempt}/${MAX_RETRIES}:`, e.message);


        if (attempt < MAX_RETRIES) {
          await sleep(PAGE_DELAY_MS * attempt);
        }
      }
    }


    throw lastError;
  }


  function collectCategoryIds(rowData) {
    return Object.entries(rowData || {})
      .filter(([key]) => /^category_ids(?:\.|$)/i.test(key))
      .map(([, value]) => value)
      .filter(v => v !== null && v !== undefined && String(v) !== '')
      .join(',');
  }


  function findCardUrl(rowData) {
    const groundId = rowData?.['Ground.id'];


    if (!groundId) {
      return { url: '', source: '' };
    }


    return {
      url: `${location.origin}/grounds/view/${groundId}`,
      source: 'Ground.id → /grounds/view/'
    };
  }


  function asValue(rowData, key, fallback = '') {
    const v = rowData?.[key];
    return v === null || v === undefined ? fallback : v;
  }


  function inspectRow(row) {
    const d = row?.rowData || {};
    const rawUrl = d['Ground.folder_url_wl'];


    if (!rawUrl) return;


    visibleUrlsSeen++;


    const host = normalizeHost(rawUrl);
    if (!host || !wanted.has(host) || found.has(host)) return;


    const card = findCardUrl(d);
    const currentStatus = asValue(d, 'Ground.current_status');
    const placementPrice = asValue(d, 'Ground.price_rur');
    const writingPrice = asValue(d, 'Ground.article_price_rur');


    found.set(host, {
      domain: host,
      status: 'НАЙДЕН В MIRALINKS',
      available_to_buy: currentStatus === 'available' ? 'ДА' : (currentStatus ? 'НЕТ/ПРОВЕРИТЬ' : ''),
      current_status: currentStatus,
      ground_id: asValue(d, 'Ground.id'),
      card_url: card.url,
      card_url_source: card.source,
      matched_url: String(rawUrl),
      placement_price_rub: placementPrice,
      writing_price_rub: writingPrice,
      total_with_writing_rub:
        Number.isFinite(Number(placementPrice)) && Number.isFinite(Number(writingPrice))
          ? Number(placementPrice) + Number(writingPrice)
          : '',
      traffic: asValue(d, 'Ground.traffic'),
      traffic_interval: asValue(d, 'traffic.interval'),
      traffic_from: asValue(d, 'traffic.from'),
      traffic_to: asValue(d, 'traffic.to'),
      articles_count: asValue(d, 'Ground.articles_count', asValue(d, 'articles_count')),
      placement_time: asValue(d, 'Ground.placement_time'),
      placement_time_text: asValue(d, 'placement_time_str'),
      last_placement: asValue(d, 'Ground.last_placement'),
      last_placement_text: asValue(d, 'last_placement_str'),
      cf: asValue(d, 'Ground.cf'),
      tf: asValue(d, 'Ground.tf'),
      ahrefs_dr: asValue(d, 'Ground.ahrefs_dr'),
      yandex_indexed_pages: asValue(d, 'Ground.ya_indexed_count'),
      region_id: asValue(d, 'Ground.region_id'),
      category_ids: collectCategoryIds(d),
      hide_folder_url: asValue(d, 'Ground.hide_folder_url')
    });


    console.log(
      `✅ ${host} | размещение ${placementPrice} ₽ | написание ${writingPrice} ₽ | ` +
      `статус ${currentStatus || '—'} | ${found.size}/${requested.length}`
    );
  }


  async function loadScope(initialParams = {}) {
    let start = Number(initialParams.iDisplayStart || 0);
    const groupValue = initialParams.sGroupValue;


    while (true) {
      if (window.__ML_STOP__) {
        throw new Error('Проверка остановлена вручную.');
      }


      const params = { iDisplayStart: start };
      if (groupValue) params.sGroupValue = groupValue;


      const data = await fetchCatalogPage(params);
      const rows = data.aaData;


      if (!rows.length) return;


      for (const row of rows) {
        const d = row?.rowData || {};
        catalogRowsSeen++;


        if (Number(d['@count'] || 0) >= 2 && d['@groupby']) {
          await sleep(PAGE_DELAY_MS);
          await loadScope({
            sGroupValue: d['@groupby'],
            iDisplayStart: 0
          });
          continue;
        }


        inspectRow(row);
      }


      console.log(
        `Miralinks v2: запросов ${requestsMade}; строк ${catalogRowsSeen}; ` +
        `видимых URL ${visibleUrlsSeen}; найдено ${found.size}/${requested.length}`
      );


      if (found.size === requested.length) return;
      if (rows.length < DISPLAY_LENGTH) return;


      start += DISPLAY_LENGTH;
      await sleep(PAGE_DELAY_MS);
    }
  }


  function csvEscape(value) {
    return `"${String(value ?? '').replace(/"/g, '""')}"`;
  }


  function downloadResults() {
    const columns = [
      'domain',
      'status',
      'available_to_buy',
      'current_status',
      'ground_id',
      'card_url',
      'card_url_source',
      'matched_url',
      'placement_price_rub',
      'writing_price_rub',
      'total_with_writing_rub',
      'traffic',
      'traffic_interval',
      'traffic_from',
      'traffic_to',
      'articles_count',
      'placement_time',
      'placement_time_text',
      'last_placement',
      'last_placement_text',
      'cf',
      'tf',
      'ahrefs_dr',
      'yandex_indexed_pages',
      'region_id',
      'category_ids',
      'hide_folder_url'
    ];


    const rows = requested.map(domain => {
      if (found.has(domain)) return found.get(domain);


      return {
        domain,
        status: 'НЕ НАЙДЕН СРЕДИ ВИДИМЫХ URL',
        available_to_buy: '',
        current_status: '',
        ground_id: '',
        card_url: '',
        card_url_source: '',
        matched_url: '',
        placement_price_rub: '',
        writing_price_rub: '',
        total_with_writing_rub: '',
        traffic: '',
        traffic_interval: '',
        traffic_from: '',
        traffic_to: '',
        articles_count: '',
        placement_time: '',
        placement_time_text: '',
        last_placement: '',
        last_placement_text: '',
        cf: '',
        tf: '',
        ahrefs_dr: '',
        yandex_indexed_pages: '',
        region_id: '',
        category_ids: '',
        hide_folder_url: ''
      };
    });


    const csv = [
      columns.map(csvEscape).join(';'),
      ...rows.map(obj =>
        columns.map(col => csvEscape(obj[col])).join(';')
      )
    ].join('\r\n');


    const blob = new Blob(
      ['\ufeff' + csv],
      { type: 'text/csv;charset=utf-8' }
    );


    const url = URL.createObjectURL(blob);
    const a = document.createElement('a');


    a.href = url;
    a.download = `miralinks_check_v2_${new Date().toISOString().slice(0, 10)}.csv`;


    document.body.appendChild(a);
    a.click();
    a.remove();


    setTimeout(() => URL.revokeObjectURL(url), 1000);
  }


  if (!location.hostname.includes('miralinks')) {
    throw new Error('Открой Miralinks → Каталог площадок и запусти скрипт там.');
  }


  console.log(`🚀 Miralinks v2. Доменов для сверки: ${requested.length}`);
  console.log('Для остановки: window.__ML_STOP__ = true');
  console.log('Используются ТЕКУЩИЕ фильтры каталога.');


  try {
    await loadScope({ iDisplayStart: 0 });
    downloadResults();


    console.log(
      `✅ V2 готово. Найдено ${found.size} из ${requested.length}. ` +
      `CSV с ценами и параметрами автоматически скачан.`
    );


  } catch (e) {
    console.error('❌ V2 остановлен:', e);
    console.log('Уже собрано:', found.size, 'из', requested.length);
    throw e;
  }
})();