diff --git a/.gitignore b/.gitignore index 9b1ee42..7b67845 100644 --- a/.gitignore +++ b/.gitignore @@ -173,3 +173,5 @@ dist # Finder (MacOS) folder config .DS_Store + +cache/ \ No newline at end of file diff --git a/cacheFetch.ts b/cacheFetch.ts new file mode 100644 index 0000000..938e867 --- /dev/null +++ b/cacheFetch.ts @@ -0,0 +1,38 @@ +// prevents TS errors +declare const self: Worker; + +const transformUrlToFile = (url: string) => { + return url.replace("https://", '').toLowerCase(); +} + +self.onmessage = async (event: MessageEvent) => { + const fileName = transformUrlToFile(event.data); + + const file = Bun.file(`./cache/${fileName}`); + + if (await file.exists()) { + const list = await file.text(); + + self.postMessage(list); + + // check if cached file is stale + + const req = await fetch(event.data); + + const text = await req.text(); + + if (text !== list) { + return await Bun.write(file, text); + } + } else { + const req = await fetch(event.data); + + const text = await req.text(); + + await Bun.write(file, text); + + self.postMessage(text); + } + + return; +}; \ No newline at end of file diff --git a/index.ts b/index.ts index b3657bf..5d7788c 100644 --- a/index.ts +++ b/index.ts @@ -1,59 +1,88 @@ const filterList = [ - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.amazon.txt", - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.apple.txt", - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.huawei.txt", - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.winoffice.txt", - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.samsung.txt", - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.tiktok.txt", - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.tiktok.extended.txt", - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.lgwebos.txt", - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.vivo.txt", - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.oppo-realme.txt", - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.xiaomi.txt", - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/fake.txt", - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/popupads.txt", - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/tif.txt", - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/tif-ips.txt", - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/pro.txt", - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/dyndns.txt", - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/hoster.txt" + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.amazon.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.apple.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.huawei.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.winoffice.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.samsung.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.tiktok.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.tiktok.extended.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.lgwebos.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.vivo.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.oppo-realme.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.xiaomi.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/fake.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/popupads.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/tif.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/tif-ips.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/pro.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/dyndns.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/hoster.txt", + "https://big.oisd.nl" ] +const cacheFetchWorker = new Worker("./cacheFetch.ts"); + +const cacheFetch = async (url: string): Promise => { + return new Promise((resolve, reject) => { + cacheFetchWorker.onmessage = (event) => { + resolve(event.data); + } + + cacheFetchWorker.onerror = (event) => { + reject(event.error); + } + + cacheFetchWorker.postMessage(url); + }) +} + +let rules: string[] = []; +let rulesText: Uint8Array; + + +const updateList = async () => { + rules = []; + for (const filter of filterList) { + let list = await cacheFetch(filter); + + list = list.replace("[Adblock Plus]", '').trim(); + list = list.replace(/^!.*$/gm, '').trim(); + + const listArray = list.split('\n') + + rules = rules.concat(listArray) + } + + rules = Array.from(new Set(rules)); + + rulesText = Bun.gzipSync(makeComment(rules.length) + rules.join('\n')) + + console.log("Updated list @", new Date().toUTCString()) +} + +const updateListInterval = setInterval(updateList, 1.8e+6) // Every 30 minutes + Bun.serve({ async fetch(request, server) { - let rules: string[] = []; - - for (const filter of filterList) { - const listReq = await fetch(filter); - let list = await listReq.text(); - - list = list.replace("[Adblock Plus]", '').trim(); - list = list.replace(/^!.*$/gm, '').trim(); - - const listArray = list.split('\n') - - rules = rules.concat(listArray) + if (new URL(request.url).pathname !== "/") { + return new Response(null, { status: 404 }); + } + + if (!rulesText) { + await updateList(); } - - const originalRuleCount = rules.length; - - rules = Array.from(new Set(rules)); - - const rulesText = makeComment(originalRuleCount, originalRuleCount - rules.length) + rules.join('\n'); return new Response(rulesText, { headers: { - "content-type": "text/plain; charset=utf-8", - "Cache-Control": "no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0", - "Pragma": "no-cache", - "Expires": "0", - "Surrogate-Control": "no-store" - }, + "Content-Type": "text/plain", + "Content-Encoding": "gzip" + } }) - } + }, + idleTimeout: 0 }) -const makeComment = (originalRuleCount: number, ruleCount: number) => { +const makeComment = (ruleCount: number) => { const now = new Date(); const second = now.getSeconds(); @@ -64,13 +93,13 @@ const makeComment = (originalRuleCount: number, ruleCount: number) => { const year = now.getFullYear(); return `[Adblock Plus] -! Title: HaGeZi's DNS Blocklists Combined -! Description: A collection of DNS blocklists by HaGeZi, combined into one list, with duplicates removed. +! Title: Combined List +! Description: A collection of dns blocklists, one list, duplicates removed. ! Expires: 1 hour ! Last Modified: ${new Date().toUTCString().split(", ")[1]} ! Version: ${year}.${month}.${day}.${hour}.${minute}.${second} ! Syntax: AdBlock -! Number of Entries: ${ruleCount} -! Original Number of Entries: ${originalRuleCount} -! Total Removed Entries: ${originalRuleCount - ruleCount}\n` -} \ No newline at end of file +! Number of Entries: ${ruleCount}\n` +} + +await updateList(); \ No newline at end of file diff --git a/test.ts b/test.ts index c8a7c36..51ad94f 100644 --- a/test.ts +++ b/test.ts @@ -1,171 +1,65 @@ -const test = { - "filters": [ - { - "url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.amazon.txt", - "name": "HaGeZi's Amazon Tracker DNS Blocklist", - "last_updated": "2025-01-14T20:51:46-05:00", - "id": 1732673071, - "rules_count": 335, - "enabled": true - }, - { - "url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.apple.txt", - "name": "HaGeZi's Apple Tracker DNS Blocklist", - "last_updated": "2025-01-14T20:51:46-05:00", - "id": 1732673072, - "rules_count": 93, - "enabled": true - }, - { - "url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.huawei.txt", - "name": "HaGeZi's Huawei Tracker DNS Blocklist", - "last_updated": "2025-01-14T20:51:46-05:00", - "id": 1732673073, - "rules_count": 95, - "enabled": true - }, - { - "url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.winoffice.txt", - "name": "HaGeZi's Windows/Office Tracker DNS Blocklist", - "last_updated": "2025-01-14T20:51:46-05:00", - "id": 1732673074, - "rules_count": 335, - "enabled": true - }, - { - "url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.samsung.txt", - "name": "HaGeZi's Samsung Tracker DNS Blocklist", - "last_updated": "2025-01-14T20:51:46-05:00", - "id": 1732673075, - "rules_count": 189, - "enabled": true - }, - { - "url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.tiktok.txt", - "name": "HaGeZi's TikTok Fingerprinting DNS Blocklist", - "last_updated": "2025-01-14T20:51:46-05:00", - "id": 1732673076, - "rules_count": 278, - "enabled": true - }, - { - "url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.tiktok.extended.txt", - "name": "HaGeZi's TikTok Extended Fingerprinting DNS Blocklist", - "last_updated": "2025-01-14T20:51:46-05:00", - "id": 1732673077, - "rules_count": 354, - "enabled": true - }, - { - "url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.lgwebos.txt", - "name": "HaGeZi's LG webOS Tracker DNS Blocklist", - "last_updated": "2025-01-14T20:51:46-05:00", - "id": 1732673078, - "rules_count": 16, - "enabled": true - }, - { - "url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.vivo.txt", - "name": "HaGeZi's Vivo Tracker DNS Blocklist", - "last_updated": "2025-01-14T20:51:46-05:00", - "id": 1732673079, - "rules_count": 88, - "enabled": true - }, - { - "url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.oppo-realme.txt", - "name": "HaGeZi's OPPO & Realme Tracker DNS Blocklist", - "last_updated": "2025-01-14T20:51:46-05:00", - "id": 1732673080, - "rules_count": 230, - "enabled": true - }, - { - "url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.xiaomi.txt", - "name": "HaGeZi's Xiaomi Tracker DNS Blocklist", - "last_updated": "2025-01-14T20:51:46-05:00", - "id": 1732673081, - "rules_count": 361, - "enabled": true - }, - { - "url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/fake.txt", - "name": "HaGeZi's Fake DNS Blocklist", - "last_updated": "2025-01-14T20:51:46-05:00", - "id": 1732673082, - "rules_count": 10389, - "enabled": true - }, - { - "url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/popupads.txt", - "name": "HaGeZi's Pop-Up Ads DNS Blocklist", - "last_updated": "2025-01-14T20:51:47-05:00", - "id": 1732673083, - "rules_count": 83577, - "enabled": true - }, - { - "url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/tif.txt", - "name": "HaGeZi's Threat Intelligence Feeds DNS Blocklist", - "last_updated": "2025-01-14T20:51:51-05:00", - "id": 1732673084, - "rules_count": 695621, - "enabled": true - }, - { - "url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/tif-ips.txt", - "name": "TIF IPs", - "last_updated": "2025-01-14T20:51:52-05:00", - "id": 1732673085, - "rules_count": 98895, - "enabled": true - }, - { - "url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/pro.txt", - "name": "HaGeZi's Pro DNS Blocklist", - "last_updated": "2025-01-14T20:51:53-05:00", - "id": 1736222077, - "rules_count": 191991, - "enabled": true - }, - { - "url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/dyndns.txt", - "name": "HaGeZi's DynDNS Blocklist", - "last_updated": "2025-01-14T20:51:53-05:00", - "id": 1736222078, - "rules_count": 1469, - "enabled": true - }, - { - "url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/hoster.txt", - "name": "HaGeZi's Badware Hoster DNS Blocklist", - "last_updated": "2025-01-14T20:51:53-05:00", - "id": 1736222079, - "rules_count": 1851, - "enabled": true - } - ], - "whitelist_filters": null, - "user_rules": [ - "" - ], - "interval": 1, - "enabled": true +const filterList = [ + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.amazon.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.apple.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.huawei.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.winoffice.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.samsung.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.tiktok.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.tiktok.extended.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.lgwebos.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.vivo.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.oppo-realme.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.xiaomi.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/fake.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/popupads.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/tif.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/tif-ips.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/pro.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/dyndns.txt", + "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/hoster.txt", + "https://big.oisd.nl" +] + +const cacheFetchWorker = new Worker("./cacheFetch.ts"); + +const cacheFetch = async (url: string): Promise => { + return new Promise((resolve, reject) => { + cacheFetchWorker.onmessage = (event) => { + resolve(event.data); + } + + cacheFetchWorker.onerror = (event) => { + reject(event.error); + } + + cacheFetchWorker.postMessage(url); + }) } -const filters = []; -for (const filter of test.filters) { - filters.push(filter.url); -} -console.log(filters); -// console.log year.month.day.hour.minute.second in 24 hour format -const now = new Date(); +/* +const fetchedLists = await Promise.all( + filterList.map((filter) => + cacheFetch(filter).then((data) => + data + .replace(/\[Adblock Plus\]/g, "") // Remove header + .replace(/^!.*$/gm, "") // Remove comments + .trim() + .split("\n") + ) + ) +);*/ -const second = now.getSeconds(); -const minute = now.getMinutes(); -const hour = now.getHours(); -const day = now.getDate(); -const month = now.getMonth() + 1; -const year = now.getFullYear(); +const fetchedLists = await Promise.all( + filterList.map(async (filter) => { + console.log("Fetching", filter); + const data = await cacheFetch(filter); + console.log("Fetched", filter); + return data + .replace(/\[Adblock Plus\]/g, "") // Remove header + .replace(/^!.*$/gm, "") // Remove comments + .trim() + .split("\n"); + }) +); -console.log(`${year}.${month}.${day}.${hour}.${minute}.${second}`); \ No newline at end of file +console.log(fetchedLists); \ No newline at end of file