seth/adguardHomeJoiner
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implement caching for DNS blocklist fetching and update list management

Browse source
This commit is contained in:
Seth 2025-01-15 14:53:12 -05:00
parent aa89875b8a
commit 79bc20aa61
No known key found for this signature in database
GPG key ID: 8B7A2C60CDF65CAC
4 changed files with 180 additions and 217 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -173,3 +173,5 @@ dist
# Finder (MacOS) folder config
.DS_Store
cache/

38
cacheFetch.ts Normal file
View file

@ -0,0 +1,38 @@
// prevents TS errors
declare const self: Worker;
const transformUrlToFile = (url: string) => {
return url.replace("https://", '').toLowerCase();
}
self.onmessage = async (event: MessageEvent) => {
const fileName = transformUrlToFile(event.data);
const file = Bun.file(`./cache/${fileName}`);
if (await file.exists()) {
const list = await file.text();
self.postMessage(list);
// check if cached file is stale
const req = await fetch(event.data);
const text = await req.text();
if (text !== list) {
return await Bun.write(file, text);
}
} else {
const req = await fetch(event.data);
const text = await req.text();
await Bun.write(file, text);
self.postMessage(text);
}
return;
};

107
index.ts
View file

@ -1,31 +1,49 @@
const filterList = [
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.amazon.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.apple.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.huawei.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.winoffice.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.samsung.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.tiktok.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.tiktok.extended.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.lgwebos.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.vivo.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.oppo-realme.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.xiaomi.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/fake.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/popupads.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/tif.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/tif-ips.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/pro.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/dyndns.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/hoster.txt"
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.amazon.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.apple.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.huawei.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.winoffice.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.samsung.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.tiktok.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.tiktok.extended.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.lgwebos.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.vivo.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.oppo-realme.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.xiaomi.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/fake.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/popupads.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/tif.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/tif-ips.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/pro.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/dyndns.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/hoster.txt",
"https://big.oisd.nl"
]
Bun.serve({
async fetch(request, server) {
let rules: string[] = [];
const cacheFetchWorker = new Worker("./cacheFetch.ts");
const cacheFetch = async (url: string): Promise<string> => {
return new Promise((resolve, reject) => {
cacheFetchWorker.onmessage = (event) => {
resolve(event.data);
}
cacheFetchWorker.onerror = (event) => {
reject(event.error);
}
cacheFetchWorker.postMessage(url);
})
}
let rules: string[] = [];
let rulesText: Uint8Array;
const updateList = async () => {
rules = [];
for (const filter of filterList) {
const listReq = await fetch(filter);
let list = await listReq.text();
let list = await cacheFetch(filter);
list = list.replace("[Adblock Plus]", '').trim();
list = list.replace(/^!.*$/gm, '').trim();
@ -35,25 +53,36 @@ Bun.serve({
rules = rules.concat(listArray)
}
const originalRuleCount = rules.length;
rules = Array.from(new Set(rules));
const rulesText = makeComment(originalRuleCount, originalRuleCount - rules.length) + rules.join('\n');
rulesText = Bun.gzipSync(makeComment(rules.length) + rules.join('\n'))
console.log("Updated list @", new Date().toUTCString())
}
const updateListInterval = setInterval(updateList, 1.8e+6) // Every 30 minutes
Bun.serve({
async fetch(request, server) {
if (new URL(request.url).pathname !== "/") {
return new Response(null, { status: 404 });
}
if (!rulesText) {
await updateList();
}
return new Response(rulesText, {
headers: {
"content-type": "text/plain; charset=utf-8",
"Cache-Control": "no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0",
"Pragma": "no-cache",
"Expires": "0",
"Surrogate-Control": "no-store"
},
})
"Content-Type": "text/plain",
"Content-Encoding": "gzip"
}
})
},
idleTimeout: 0
})
const makeComment = (originalRuleCount: number, ruleCount: number) => {
const makeComment = (ruleCount: number) => {
const now = new Date();
const second = now.getSeconds();
@ -64,13 +93,13 @@ const makeComment = (originalRuleCount: number, ruleCount: number) => {
const year = now.getFullYear();
return `[Adblock Plus]
! Title: HaGeZi's DNS Blocklists Combined
! Description: A collection of DNS blocklists by HaGeZi, combined into one list, with duplicates removed.
! Title: Combined List
! Description: A collection of dns blocklists, one list, duplicates removed.
! Expires: 1 hour
! Last Modified: ${new Date().toUTCString().split(", ")[1]}
! Version: ${year}.${month}.${day}.${hour}.${minute}.${second}
! Syntax: AdBlock
! Number of Entries: ${ruleCount}
! Original Number of Entries: ${originalRuleCount}
! Total Removed Entries: ${originalRuleCount - ruleCount}\n`
! Number of Entries: ${ruleCount}\n`
}
await updateList();

230
test.ts
View file

@ -1,171 +1,65 @@
const test = {
"filters": [
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.amazon.txt",
"name": "HaGeZi's Amazon Tracker DNS Blocklist",
"last_updated": "2025-01-14T20:51:46-05:00",
"id": 1732673071,
"rules_count": 335,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.apple.txt",
"name": "HaGeZi's Apple Tracker DNS Blocklist",
"last_updated": "2025-01-14T20:51:46-05:00",
"id": 1732673072,
"rules_count": 93,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.huawei.txt",
"name": "HaGeZi's Huawei Tracker DNS Blocklist",
"last_updated": "2025-01-14T20:51:46-05:00",
"id": 1732673073,
"rules_count": 95,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.winoffice.txt",
"name": "HaGeZi's Windows/Office Tracker DNS Blocklist",
"last_updated": "2025-01-14T20:51:46-05:00",
"id": 1732673074,
"rules_count": 335,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.samsung.txt",
"name": "HaGeZi's Samsung Tracker DNS Blocklist",
"last_updated": "2025-01-14T20:51:46-05:00",
"id": 1732673075,
"rules_count": 189,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.tiktok.txt",
"name": "HaGeZi's TikTok Fingerprinting DNS Blocklist",
"last_updated": "2025-01-14T20:51:46-05:00",
"id": 1732673076,
"rules_count": 278,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.tiktok.extended.txt",
"name": "HaGeZi's TikTok Extended Fingerprinting DNS Blocklist",
"last_updated": "2025-01-14T20:51:46-05:00",
"id": 1732673077,
"rules_count": 354,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.lgwebos.txt",
"name": "HaGeZi's LG webOS Tracker DNS Blocklist",
"last_updated": "2025-01-14T20:51:46-05:00",
"id": 1732673078,
"rules_count": 16,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.vivo.txt",
"name": "HaGeZi's Vivo Tracker DNS Blocklist",
"last_updated": "2025-01-14T20:51:46-05:00",
"id": 1732673079,
"rules_count": 88,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.oppo-realme.txt",
"name": "HaGeZi's OPPO & Realme Tracker DNS Blocklist",
"last_updated": "2025-01-14T20:51:46-05:00",
"id": 1732673080,
"rules_count": 230,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.xiaomi.txt",
"name": "HaGeZi's Xiaomi Tracker DNS Blocklist",
"last_updated": "2025-01-14T20:51:46-05:00",
"id": 1732673081,
"rules_count": 361,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/fake.txt",
"name": "HaGeZi's Fake DNS Blocklist",
"last_updated": "2025-01-14T20:51:46-05:00",
"id": 1732673082,
"rules_count": 10389,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/popupads.txt",
"name": "HaGeZi's Pop-Up Ads DNS Blocklist",
"last_updated": "2025-01-14T20:51:47-05:00",
"id": 1732673083,
"rules_count": 83577,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/tif.txt",
"name": "HaGeZi's Threat Intelligence Feeds DNS Blocklist",
"last_updated": "2025-01-14T20:51:51-05:00",
"id": 1732673084,
"rules_count": 695621,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/tif-ips.txt",
"name": "TIF IPs",
"last_updated": "2025-01-14T20:51:52-05:00",
"id": 1732673085,
"rules_count": 98895,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/pro.txt",
"name": "HaGeZi's Pro DNS Blocklist",
"last_updated": "2025-01-14T20:51:53-05:00",
"id": 1736222077,
"rules_count": 191991,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/dyndns.txt",
"name": "HaGeZi's DynDNS Blocklist",
"last_updated": "2025-01-14T20:51:53-05:00",
"id": 1736222078,
"rules_count": 1469,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/hoster.txt",
"name": "HaGeZi's Badware Hoster DNS Blocklist",
"last_updated": "2025-01-14T20:51:53-05:00",
"id": 1736222079,
"rules_count": 1851,
"enabled": true
}
],
"whitelist_filters": null,
"user_rules": [
""
],
"interval": 1,
"enabled": true
}
const filters = [];
for (const filter of test.filters) {
filters.push(filter.url);
}
console.log(filters);
const filterList = [
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.amazon.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.apple.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.huawei.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.winoffice.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.samsung.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.tiktok.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.tiktok.extended.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.lgwebos.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.vivo.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.oppo-realme.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.xiaomi.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/fake.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/popupads.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/tif.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/tif-ips.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/pro.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/dyndns.txt",
"https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/hoster.txt",
"https://big.oisd.nl"
]
// console.log year.month.day.hour.minute.second in 24 hour format
const now = new Date();
const cacheFetchWorker = new Worker("./cacheFetch.ts");
const second = now.getSeconds();
const minute = now.getMinutes();
const hour = now.getHours();
const day = now.getDate();
const month = now.getMonth() + 1;
const year = now.getFullYear();
const cacheFetch = async (url: string): Promise<string> => {
return new Promise((resolve, reject) => {
cacheFetchWorker.onmessage = (event) => {
resolve(event.data);
}
console.log(`${year}.${month}.${day}.${hour}.${minute}.${second}`);
cacheFetchWorker.onerror = (event) => {
reject(event.error);
}
cacheFetchWorker.postMessage(url);
})
}
/*
const fetchedLists = await Promise.all(
filterList.map((filter) =>
cacheFetch(filter).then((data) =>
data
.replace(/\[Adblock Plus\]/g, "") // Remove header
.replace(/^!.*$/gm, "") // Remove comments
.trim()
.split("\n")
)
)
);*/
const fetchedLists = await Promise.all(
filterList.map(async (filter) => {
console.log("Fetching", filter);
const data = await cacheFetch(filter);
console.log("Fetched", filter);
return data
.replace(/\[Adblock Plus\]/g, "") // Remove header
.replace(/^!.*$/gm, "") // Remove comments
.trim()
.split("\n");
})
);
console.log(fetchedLists);