// ==UserScript==
// @name Bypass Paywalls Clean - nl/be
// @version 4.0.8.2
// @description Bypass Paywalls of news sites
// @author magnolia1234
// @downloadURL https://gitflic.ru/project/magnolia1234/bypass-paywalls-clean-filters/blob/raw?file=userscript/bpc.nl.user.js
// @updateURL https://gitflic.ru/project/magnolia1234/bypass-paywalls-clean-filters/blob/raw?file=userscript/bpc.nl.user.js
// @homepageURL https://gitflic.ru/project/magnolia1234/bypass-paywalls-clean-filters
// @supportURL https://gitflic.ru/project/magnolia1234/bypass-paywalls-clean-filters
// @license MIT; https://gitflic.ru/project/magnolia1234/bypass-paywalls-clean-filters/blob/raw?file=LICENSE
// @match *://*.nl/*
// @match *://*.artsenkrant.com/*
// @match *://*.businessam.be/*
// @match *://*.demorgen.be/*
// @match *://*.doorbraak.be/*
// @match *://*.flair.be/nl/*
// @match *://*.gva.be/*
// @match *://*.hbvl.be/*
// @match *://*.hln.be/*
// @match *://*.humo.be/*
// @match *://*.knack.be/*
// @match *://*.kw.be/*
// @match *://*.libelle.be/*
// @match *://*.nieuwsblad.be/*
// @match *://*.projectcargojournal.com/*
// @match *://*.railfreight.cn/*
// @match *://*.railfreight.com/*
// @match *://*.railtech.be/*
// @match *://*.railtech.com/*
// @match *://*.standaard.be/*
// @match *://*.taxipro.be/*
// @connect archive.fo
// @connect archive.is
// @connect archive.li
// @connect archive.md
// @connect archive.ph
// @connect archive.vn
// @grant GM.xmlHttpRequest
// ==/UserScript==
(function() {
//'use strict';
var func_post;
var fetch_headers = {};
window.setTimeout(function () {
var domain;
var mobile = window.navigator.userAgent.toLowerCase().includes('mobile');
var csDoneOnce;
var cs_param = {};
var overlay = document.querySelector('body.didomi-popup-open');
if (overlay)
overlay.classList.remove('didomi-popup-open');
var ads = 'div.OUTBRAIN, div[id^="taboola-"], div.ad-container, div[class*="-ad-container"], div[class*="_ad-container"], div.arc_ad, div[id^="poool-"], amp-ad, amp-embed[type="mgid"], amp-embed[type="outbrain"], amp-embed[type="taboola"]';
hideDOMStyle(ads, 10);
var be_mediahuis_domains = ['gva.be', 'hbvl.be', 'nieuwsblad.be', 'standaard.be'];
var be_roularta_domains = ['artsenkrant.com', 'beleggersbelangen.nl', 'flair.be', 'knack.be', 'kw.be', 'libelle.be'];
var nl_dpg_adr_domains = ['ad.nl', 'bd.nl', 'bndestem.nl', 'destentor.nl', 'ed.nl', 'gelderlander.nl', 'pzc.nl', 'tubantia.nl'];
var nl_dpg_media_domains = ['demorgen.be', 'flair.nl', 'humo.be', 'libelle.nl', 'margriet.nl', 'parool.nl', 'trouw.nl', 'volkskrant.nl'];
if (matchDomain(be_mediahuis_domains)) {
window.setTimeout(function () {
let video = document.querySelector('div.video');
func_post = function () {
if (video) {
let video_new = document.querySelector('div[id$="-streamone"], div[id^="video-player-"]') || document.querySelector('article div[style^="background-color"]');
if (!video_new) {
let empty_div = document.querySelector('div > div[aria-hidden="true"]:empty:not([class])');
if (empty_div)
video_new = empty_div.parentNode;
}
if (video_new && video_new.parentNode)
video_new.parentNode.replaceChild(video, video_new);
}
if (mobile) {
let lazy_images = document.querySelectorAll('figure img[loading="lazy"][style]');
for (let elem of lazy_images)
elem.style = 'width: 95%;';
let figures = document.querySelectorAll('figure div');
for (let elem of figures) {
elem.removeAttribute('style');
let svg = elem.querySelector('svg');
removeDOMElement(svg);
}
}
let article = document.querySelector(article_sel);
if (article && !matchDomain(['gva.be', 'hbvl.be'])) {
let pars = article.querySelectorAll('div[style*="font-size"]');
if (pars.length < 5)
article.before(googleSearchToolLink(url));
}
}
let url = window.location.href;
let paywall_sel = 'head > meta[name$="article_ispaidcontent"][content="true"]';
let paywall = document.querySelector(paywall_sel);
let art_div_sel = 'div[id]:not([id^="warning"]):not([id^="ad_"]):not([id^="webshop-"])';
let article_sel = 'article ' + art_div_sel + ', section ' + art_div_sel + ', main ' + art_div_sel + ', div[class^="article-body_"]';
if (paywall) {
getArchive(url, paywall_sel, '', article_sel);
if (matchDomain(['gva.be', 'hbvl.be'])) {
let article = document.querySelector(article_sel);
if (article)
article.before(googleSearchToolLink(url));
}
}
let popup = document.querySelector('div[data-testid="close-popup-button"]');
if (popup)
popup.click();
}, 1500);
}
else if (matchDomain('businessam.be')) {
let paywall = document.querySelector('div.paywall');
if (paywall) {
removeDOMElement(paywall);
let article = document.querySelector('div.text-gradient');
if (article) {
let scripts = document.querySelectorAll('script:not([src]):not([type])');
let content_script;
for (let script of scripts) {
if (script.text.match(/window\.fullcontent64\s?=\s?"/)) {
content_script = script;
break;
}
}
if (content_script) {
try {
let content = decode_utf8(atob(content_script.text.split(/window\.fullcontent64\s?=\s?"/)[1].split('";')[0]));
let parser = new DOMParser();
let doc = parser.parseFromString('
'));
let json_text = json[0];
if (json_text) {
let parser = new DOMParser();
let doc = parser.parseFromString('
')[0].split('href="')[1].split('"')[0];
getArticleSrc(url, url_src, proxy, base64, selector, text_fail, selector_source, selector_archive);
recursive = true;
} else
html = '';
}
}
if (!recursive)
replaceDomElementExtSrc(url, url_src, html, proxy, base64, selector, text_fail, selector_source, selector_archive);
}
});
}
function replaceDomElementExt(url, proxy, base64, selector, text_fail = '', selector_source = selector, selector_archive = selector) {
let article = document.querySelector(selector);
if (!article)
return;
if (proxy) {
if (!text_fail) {
if (url.startsWith('https://archive.'))
text_fail = 'BPC > Try for full article text (no need to report issue for external site):\r\n';
else if (!matchUrlDomain(window.location.hostname, url))
text_fail = 'BPC > failed to load from external site:\r\n';
}
getArticleSrc(url, '', proxy, base64, selector, text_fail, selector_source, selector_archive);
} else {
fetch(url, {headers: fetch_headers})
.then(response => {
let article = document.querySelector(selector);
if (response.ok) {
response.text().then(html => {
replaceDomElementExtSrc(url, '', html, false, base64, selector, text_fail, selector_source);
});
} else {
replaceTextFail(url, article, proxy, text_fail);
}
}).catch(function (err) {
replaceTextFail(url, article, proxy, text_fail);
});
}
}
var selector_level = false;
function replaceDomElementExtSrc(url, url_src, html, proxy, base64, selector, text_fail = '', selector_source = selector, selector_archive = selector) {
let article = document.querySelector(selector);
let article_link = document.querySelector(selector_archive);
let no_content_msg = ' | no article content found! | :';
if (html) {
if (!proxy && base64) {
html = decode_utf8(atob(html));
selector_source = 'body';
}
let parser = new DOMParser();
window.setTimeout(function () {
if (url.startsWith('https://archive.') && url_src) {
let domain_archive = url.match(/^https:\/\/(archive\.\w{2})/)[1];
let pathname = new URL(url_src).pathname;
html = html.replace(new RegExp('https:\\/\\/' + domain_archive.replace('.', '\\.') + '\\/o\\/\\w+\\/', 'g'), '').replace(new RegExp("(src=\"|background-image:url\\(')" + pathname.replace('/', '\\/'), 'g'), "$1" + 'https://' + domain_archive + pathname);
}
let doc = parser.parseFromString(html, 'text/html');
if (selector_level)
selector_source = getSelectorLevel(selector_source);
let article_new = doc.querySelector(selector_source);
if (article_new) {
if (article && article.parentNode) {
if (url.startsWith('https://archive.')) {
let arch_dom = (selector_archive !== selector) ? (article_new.querySelector(selector_archive) || document.querySelector(selector_archive)) : article_new;
if (arch_dom) {
if (arch_dom.firstChild)
arch_dom = arch_dom.firstChild;
let arch_div = document.createElement('div');
arch_div.appendChild(archiveLink_renew(url_src));
arch_div.appendChild(archiveLink(window.location.href, 'BPC > Full article text fetched from (no need to report issue for external site):\r\n'));
arch_div.style = 'margin: 0px 0px 50px;';
arch_dom.before(arch_div);
}
let targets = article_new.querySelectorAll('a[target="_blank"][href^="' + window.location.origin + '"]');
for (let elem of targets)
elem.removeAttribute('target');
let invalid_links = article_new.querySelectorAll('link[rel*="preload"]:not([href])');
removeDOMElement(...invalid_links);
}
window.setTimeout(function () {
if (article.parentNode) {
article.parentNode.replaceChild(article_new, article);
if (func_post)
func_post();
}
}, 200);
}
} else
replaceTextFail(url, article_link, proxy, text_fail.replace(':', no_content_msg));
}, 200);
} else {
replaceTextFail(url, article_link, proxy, url_src ? text_fail.replace(':', no_content_msg) : text_fail);
}
}
function replaceTextFail(url, article, proxy, text_fail) {
if (text_fail && article) {
let text_fail_div = document.createElement('div');
text_fail_div.id = 'bpc_fail';
text_fail_div.setAttribute('style', 'margin: 0px 50px; font-weight: bold; color: red;');
text_fail_div.appendChild(document.createTextNode(text_fail));
if (proxy) {
if (url.startsWith('https://archive.')) {
text_fail_div = archiveLink(url.replace(/^https:\/\/archive\.\w{2}\//, ''), text_fail);
} else {
let a_link = document.createElement('a');
a_link.innerText = url;
a_link.href = url;
a_link.target = '_blank';
text_fail_div.appendChild(a_link);
}
}
if (article.firstChild)
article.firstChild.before(text_fail_div);
else
article.appendChild(text_fail_div);
}
}
function randomInt(max) {
return Math.floor(Math.random() * Math.floor(max));
}
function archiveRandomDomain() {
let tld_array = ['fo', 'is', 'li', 'md', 'ph', 'vn'];
let tld = tld_array[randomInt(6)];
return 'archive.' + tld;
}
function getArchive(url, paywall_sel, paywall_action = '', selector, text_fail = '', selector_source = selector, selector_archive = selector) {
let url_archive = 'https://' + archiveRandomDomain() + '/' + url.split(/[#\?]/)[0];
let paywall = document.querySelectorAll(paywall_sel);
if (paywall.length) {
clearPaywall(paywall, paywall_action);
replaceDomElementExt(url_archive, true, false, selector, text_fail, selector_source, selector_archive);
}
}
function archiveLink(url, text_fail = 'BPC > Try for full article text (no need to report issue for external site):\r\n') {
return externalLink(['archive.today', archiveRandomDomain()], 'https://{domain}?run=1&url={url}', url, text_fail);
}
function archiveLink_renew(url, text_fail = 'BPC > Only use to renew if text is incomplete or updated:\r\n') {
return externalLink([new URL(url).hostname], '{url}/again?url=' + window.location.href, url, text_fail);
}
function nftLink(url, text_fail = 'BPC > Full article text:\r\n') {
return externalLink(['1ft.io'], 'https://{domain}/{url}', url, text_fail);
}
function externalLink(domains, ext_url_templ, url, text_fail = 'BPC > Full article text:\r\n') {
let text_fail_div = document.createElement('div');
text_fail_div.id = 'bpc_archive';
text_fail_div.setAttribute('style', 'margin: 20px; font-size: 20px; font-weight: bold; color: red;');
let parser = new DOMParser();
text_fail = text_fail.replace(/\[(?
[^\]]+)\]/g, function (match, url) {
return "" + new URL(url).hostname + "";
});
let doc = parser.parseFromString('' + text_fail + '', 'text/html');
let elem = doc.querySelector('span');
text_fail_div.appendChild(elem);
for (let domain of domains) {
let ext_url = ext_url_templ.replace('{domain}', domain).replace('{url}', url.split('?')[0]);
let a_link = document.createElement('a');
a_link.innerText = domain;
a_link.href = ext_url;
a_link.target = '_blank';
text_fail_div.appendChild(document.createTextNode(' | '));
text_fail_div.appendChild(a_link);
}
return text_fail_div;
}
function getArticleJsonScript() {
let scripts = document.querySelectorAll('script[type="application/ld+json"]');
let json_script;
for (let script of scripts) {
if (script.innerText.match(/"(articlebody|text)":/i)) {
json_script = script;
break;
}
}
return json_script;
}
function getNestedKeys(obj, key) {
if (key in obj)
return obj[key];
let keys = key.split('.');
let value = obj;
for (let i = 0; i < keys.length; i++) {
value = value[keys[i]];
if (value === undefined)
break;
}
return value;
}
function getJsonUrlText(article, callback, article_id = '', key = '', url_rest = false, url_slash = false) {
let json_url_dom = document.querySelector('head > link[rel="alternate"][type="application/json"][href]');
let json_url;
if (json_url_dom)
json_url = json_url_dom.href;
if (!json_url && article_id)
json_url = window.location.origin + '/wp-json/wp/v2/posts/' + article_id;
if (url_rest)
json_url = json_url.replace('/wp-json/', '/?rest_route=/');
else if (url_slash)
json_url = json_url.replace('/wp-json/', '//wp-json/');
if (json_url) {
fetch(json_url)
.then(response => {
if (response.ok) {
response.json().then(json => {
try {
let json_text = parseHtmlEntities(!key ? json.content.rendered : getNestedKeys(json, key));
callback(json_text, article);
} catch (err) {
console.log(err);
}
});
}
});
}
}
function getJsonUrlAdd(json_text, article, art_options = {}) {
let art_type = 'div';
let art_attrib = '';
if (Object.keys(art_options).length) {
if (art_options.art_type)
art_type = art_options.art_type;
if (art_options.art_class)
art_attrib += ' class="' + art_options.art_class + '"';
if (art_options.art_id)
art_attrib += ' id="' + art_options.art_id + '"';
if (art_options.art_style)
art_attrib += ' style="' + art_options.art_style + '"';
if (art_options.func_text)
json_text = art_options.func_text(json_text);
}
let parser = new DOMParser();
let doc = parser.parseFromString('<' + art_type + art_attrib + '>' + json_text + '' + art_type + '>', 'text/html');
let article_new = doc.querySelector(art_type);
if (art_options.art_append || !article.parentNode) {
if (!art_options.art_hold)
article.innerHTML = '';
article.appendChild(article_new);
} else
article.parentNode.replaceChild(article_new, article);
if (func_post)
func_post();
}
function getJsonUrl(paywall_sel, paywall_action = '', article_sel, art_options = {}, article_id = '', key = '', url_rest = false, url_slash = false) {
let paywall = document.querySelectorAll(paywall_sel);
let article = document.querySelector(article_sel);
if (paywall.length && article) {
clearPaywall(paywall, paywall_action);
getJsonUrlText(article, (json_text, article) => {
if (json_text && article)
getJsonUrlAdd(json_text, article, art_options);
}, article_id, key, url_rest, url_slash);
}
}
function header_nofix(header, cond_sel = '', msg = 'BPC > no fix') {
if (header && typeof header === 'string')
header = document.querySelector(header);
if (header && !document.querySelector('div#bpc_nofix')) {
if (cond_sel) {
let elem = document.querySelectorAll(cond_sel);
if (elem.length)
removeDOMElement(...elem);
else
return false;
}
let nofix_div = document.createElement('div');
nofix_div.id = 'bpc_nofix';
nofix_div.style = 'margin: 20px; font-size: 20px; font-weight: bold; color: red;';
nofix_div.innerText = msg;
header.before(nofix_div);
}
}
function pageContains(selector, text) {
let elements = document.querySelectorAll(selector);
return Array.prototype.filter.call(elements, function (element) {
return RegExp(text).test(element.textContent);
});
}
function parseHtmlEntities(encodedString) {
let translate_re = /&(nbsp|amp|quot|lt|gt|deg|hellip|laquo|raquo|ldquo|rdquo|lsquo|rsquo|mdash);/g;
let translate = {"nbsp": " ", "amp": "&", "quot": "\"", "lt": "<", "gt": ">", "deg": "°", "hellip": "…",
"laquo": "«", "raquo": "»", "ldquo": "“", "rdquo": "”", "lsquo": "‘", "rsquo": "’", "mdash": "—"};
return encodedString.replace(translate_re, function (match, entity) {
return translate[entity];
}).replace(/(\d+);/gi, function (match, numStr) {
let num = parseInt(numStr, 10);
return String.fromCharCode(num);
});
}
function encode_utf8(str) {
return unescape(encodeURIComponent(str));
}
function decode_utf8(str) {
return decodeURIComponent(escape(str));
}
function ampToHtml() {
window.setTimeout(function () {
let canonical = document.querySelector('head > link[rel="canonical"][href]');
if (canonical)
window.location.href = canonical.href;
}, 1000);
}
function check_loop(interval = 2000) {
let loop = true;
let loop_date = Number(sessionStorage.getItem('###_loop'));
if (!(loop_date && (Date.now() - loop_date < interval))) {
sessionStorage.setItem('###_loop', Date.now());
loop = false;
}
return loop;
}
function refreshCurrentTab(not_loop = true) {
if (!not_loop || !check_loop(5000)) {
window.setTimeout(function () {
window.location.reload(true);
}, 500);
} else {
let header = (document.body && document.body.firstChild) || document.documentElement;
header_nofix(header, '', 'BPC > refresh loop stopped');
}
}
})();