seth/adguardHomeJoiner
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implement caching for DNS blocklist fetching and update list management

Browse source
This commit is contained in:
Seth 2025-01-15 14:53:12 -05:00
parent aa89875b8a
commit 79bc20aa61
No known key found for this signature in database
GPG key ID: 8B7A2C60CDF65CAC
4 changed files with 180 additions and 217 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -173,3 +173,5 @@ dist
# Finder (MacOS) folder config # Finder (MacOS) folder config
.DS_Store .DS_Store
cache/

38
cacheFetch.ts Normal file
View file

@ -0,0 +1,38 @@
// prevents TS errors
declare const self: Worker;
const transformUrlToFile = (url: string) => {
return url.replace("https://", '').toLowerCase();
}
self.onmessage = async (event: MessageEvent) => {
const fileName = transformUrlToFile(event.data);
const file = Bun.file(`./cache/${fileName}`);
if (await file.exists()) {
const list = await file.text();
self.postMessage(list);
// check if cached file is stale
const req = await fetch(event.data);
const text = await req.text();
if (text !== list) {
return await Bun.write(file, text);
}
} else {
const req = await fetch(event.data);
const text = await req.text();
await Bun.write(file, text);
self.postMessage(text);
}
return;
};

129
index.ts
View file

@ -1,59 +1,88 @@
const filterList = [ const filterList = [
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.amazon.txt", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.amazon.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.apple.txt", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.apple.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.huawei.txt", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.huawei.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.winoffice.txt", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.winoffice.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.samsung.txt", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.samsung.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.tiktok.txt", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.tiktok.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.tiktok.extended.txt", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.tiktok.extended.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.lgwebos.txt", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.lgwebos.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.vivo.txt", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.vivo.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.oppo-realme.txt", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.oppo-realme.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.xiaomi.txt", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.xiaomi.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/fake.txt", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/fake.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/popupads.txt", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/popupads.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/tif.txt", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/tif.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/tif-ips.txt", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/tif-ips.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/pro.txt", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/pro.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/dyndns.txt", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/dyndns.txt",
"https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/hoster.txt" "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/hoster.txt",
"https://big.oisd.nl"
] ]
const cacheFetchWorker = new Worker("./cacheFetch.ts");
const cacheFetch = async (url: string): Promise<string> => {
return new Promise((resolve, reject) => {
cacheFetchWorker.onmessage = (event) => {
resolve(event.data);
}
cacheFetchWorker.onerror = (event) => {
reject(event.error);
}
cacheFetchWorker.postMessage(url);
})
}
let rules: string[] = [];
let rulesText: Uint8Array;
const updateList = async () => {
rules = [];
for (const filter of filterList) {
let list = await cacheFetch(filter);
list = list.replace("[Adblock Plus]", '').trim();
list = list.replace(/^!.*$/gm, '').trim();
const listArray = list.split('\n')
rules = rules.concat(listArray)
}
rules = Array.from(new Set(rules));
rulesText = Bun.gzipSync(makeComment(rules.length) + rules.join('\n'))
console.log("Updated list @", new Date().toUTCString())
}
const updateListInterval = setInterval(updateList, 1.8e+6) // Every 30 minutes
Bun.serve({ Bun.serve({
async fetch(request, server) { async fetch(request, server) {
let rules: string[] = []; if (new URL(request.url).pathname !== "/") {
return new Response(null, { status: 404 });
for (const filter of filterList) { }
const listReq = await fetch(filter);
let list = await listReq.text(); if (!rulesText) {
await updateList();
list = list.replace("[Adblock Plus]", '').trim();
list = list.replace(/^!.*$/gm, '').trim();
const listArray = list.split('\n')
rules = rules.concat(listArray)
} }
const originalRuleCount = rules.length;
rules = Array.from(new Set(rules));
const rulesText = makeComment(originalRuleCount, originalRuleCount - rules.length) + rules.join('\n');
return new Response(rulesText, { return new Response(rulesText, {
headers: { headers: {
"content-type": "text/plain; charset=utf-8", "Content-Type": "text/plain",
"Cache-Control": "no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0", "Content-Encoding": "gzip"
"Pragma": "no-cache", }
"Expires": "0",
"Surrogate-Control": "no-store"
},
}) })
} },
idleTimeout: 0
}) })
const makeComment = (originalRuleCount: number, ruleCount: number) => { const makeComment = (ruleCount: number) => {
const now = new Date(); const now = new Date();
const second = now.getSeconds(); const second = now.getSeconds();
@ -64,13 +93,13 @@ const makeComment = (originalRuleCount: number, ruleCount: number) => {
const year = now.getFullYear(); const year = now.getFullYear();
return `[Adblock Plus] return `[Adblock Plus]
! Title: HaGeZi's DNS Blocklists Combined ! Title: Combined List
! Description: A collection of DNS blocklists by HaGeZi, combined into one list, with duplicates removed. ! Description: A collection of dns blocklists, one list, duplicates removed.
! Expires: 1 hour ! Expires: 1 hour
! Last Modified: ${new Date().toUTCString().split(", ")[1]} ! Last Modified: ${new Date().toUTCString().split(", ")[1]}
! Version: ${year}.${month}.${day}.${hour}.${minute}.${second} ! Version: ${year}.${month}.${day}.${hour}.${minute}.${second}
! Syntax: AdBlock ! Syntax: AdBlock
! Number of Entries: ${ruleCount} ! Number of Entries: ${ruleCount}\n`
! Original Number of Entries: ${originalRuleCount} }
! Total Removed Entries: ${originalRuleCount - ruleCount}\n`
} await updateList();

228
test.ts
View file

@ -1,171 +1,65 @@
const test = { const filterList = [
"filters": [ "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.amazon.txt",
{ "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.apple.txt",
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.amazon.txt", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.huawei.txt",
"name": "HaGeZi's Amazon Tracker DNS Blocklist", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.winoffice.txt",
"last_updated": "2025-01-14T20:51:46-05:00", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.samsung.txt",
"id": 1732673071, "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.tiktok.txt",
"rules_count": 335, "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.tiktok.extended.txt",
"enabled": true "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.lgwebos.txt",
}, "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.vivo.txt",
{ "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.oppo-realme.txt",
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.apple.txt", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/native.xiaomi.txt",
"name": "HaGeZi's Apple Tracker DNS Blocklist", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/fake.txt",
"last_updated": "2025-01-14T20:51:46-05:00", "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/popupads.txt",
"id": 1732673072, "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/tif.txt",
"rules_count": 93, "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/tif-ips.txt",
"enabled": true "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/pro.txt",
}, "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/dyndns.txt",
{ "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@main/adblock/hoster.txt",
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.huawei.txt", "https://big.oisd.nl"
"name": "HaGeZi's Huawei Tracker DNS Blocklist", ]
"last_updated": "2025-01-14T20:51:46-05:00",
"id": 1732673073, const cacheFetchWorker = new Worker("./cacheFetch.ts");
"rules_count": 95,
"enabled": true const cacheFetch = async (url: string): Promise<string> => {
}, return new Promise((resolve, reject) => {
{ cacheFetchWorker.onmessage = (event) => {
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.winoffice.txt", resolve(event.data);
"name": "HaGeZi's Windows/Office Tracker DNS Blocklist", }
"last_updated": "2025-01-14T20:51:46-05:00",
"id": 1732673074, cacheFetchWorker.onerror = (event) => {
"rules_count": 335, reject(event.error);
"enabled": true }
},
{ cacheFetchWorker.postMessage(url);
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.samsung.txt", })
"name": "HaGeZi's Samsung Tracker DNS Blocklist",
"last_updated": "2025-01-14T20:51:46-05:00",
"id": 1732673075,
"rules_count": 189,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.tiktok.txt",
"name": "HaGeZi's TikTok Fingerprinting DNS Blocklist",
"last_updated": "2025-01-14T20:51:46-05:00",
"id": 1732673076,
"rules_count": 278,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.tiktok.extended.txt",
"name": "HaGeZi's TikTok Extended Fingerprinting DNS Blocklist",
"last_updated": "2025-01-14T20:51:46-05:00",
"id": 1732673077,
"rules_count": 354,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.lgwebos.txt",
"name": "HaGeZi's LG webOS Tracker DNS Blocklist",
"last_updated": "2025-01-14T20:51:46-05:00",
"id": 1732673078,
"rules_count": 16,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.vivo.txt",
"name": "HaGeZi's Vivo Tracker DNS Blocklist",
"last_updated": "2025-01-14T20:51:46-05:00",
"id": 1732673079,
"rules_count": 88,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.oppo-realme.txt",
"name": "HaGeZi's OPPO & Realme Tracker DNS Blocklist",
"last_updated": "2025-01-14T20:51:46-05:00",
"id": 1732673080,
"rules_count": 230,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/native.xiaomi.txt",
"name": "HaGeZi's Xiaomi Tracker DNS Blocklist",
"last_updated": "2025-01-14T20:51:46-05:00",
"id": 1732673081,
"rules_count": 361,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/fake.txt",
"name": "HaGeZi's Fake DNS Blocklist",
"last_updated": "2025-01-14T20:51:46-05:00",
"id": 1732673082,
"rules_count": 10389,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/popupads.txt",
"name": "HaGeZi's Pop-Up Ads DNS Blocklist",
"last_updated": "2025-01-14T20:51:47-05:00",
"id": 1732673083,
"rules_count": 83577,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/tif.txt",
"name": "HaGeZi's Threat Intelligence Feeds DNS Blocklist",
"last_updated": "2025-01-14T20:51:51-05:00",
"id": 1732673084,
"rules_count": 695621,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/tif-ips.txt",
"name": "TIF IPs",
"last_updated": "2025-01-14T20:51:52-05:00",
"id": 1732673085,
"rules_count": 98895,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/pro.txt",
"name": "HaGeZi's Pro DNS Blocklist",
"last_updated": "2025-01-14T20:51:53-05:00",
"id": 1736222077,
"rules_count": 191991,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/dyndns.txt",
"name": "HaGeZi's DynDNS Blocklist",
"last_updated": "2025-01-14T20:51:53-05:00",
"id": 1736222078,
"rules_count": 1469,
"enabled": true
},
{
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/hoster.txt",
"name": "HaGeZi's Badware Hoster DNS Blocklist",
"last_updated": "2025-01-14T20:51:53-05:00",
"id": 1736222079,
"rules_count": 1851,
"enabled": true
}
],
"whitelist_filters": null,
"user_rules": [
""
],
"interval": 1,
"enabled": true
} }
const filters = [];
for (const filter of test.filters) {
filters.push(filter.url);
}
console.log(filters);
// console.log year.month.day.hour.minute.second in 24 hour format /*
const now = new Date(); const fetchedLists = await Promise.all(
filterList.map((filter) =>
cacheFetch(filter).then((data) =>
data
.replace(/\[Adblock Plus\]/g, "") // Remove header
.replace(/^!.*$/gm, "") // Remove comments
.trim()
.split("\n")
)
)
);*/
const second = now.getSeconds(); const fetchedLists = await Promise.all(
const minute = now.getMinutes(); filterList.map(async (filter) => {
const hour = now.getHours(); console.log("Fetching", filter);
const day = now.getDate(); const data = await cacheFetch(filter);
const month = now.getMonth() + 1; console.log("Fetched", filter);
const year = now.getFullYear(); return data
.replace(/\[Adblock Plus\]/g, "") // Remove header
.replace(/^!.*$/gm, "") // Remove comments
.trim()
.split("\n");
})
);
console.log(`${year}.${month}.${day}.${hour}.${minute}.${second}`); console.log(fetchedLists);