mirror of
https://github.com/aclist/dztui.git
synced 2025-07-08 08:48:59 +02:00
Merge pull request #216 from aclist/feat/fetch_ipdb
feat: auto fetch geolocation records
This commit is contained in:
aclist
GitHub
commit
74bfdf42ae
2 changed files with 137 additions and 31 deletions
38
CHANGELOG.md
38
CHANGELOG.md
|
|
@ -1,18 +1,28 @@
|
|||
# Changelog
|
||||
|
||||
## [5.7.0-beta.13] 2024-04-17
|
||||
## [5.8.0-beta.1] 2025-06-05
|
||||
## Added
|
||||
- Automatically fetch geolocation records
|
||||
## Fixed
|
||||
- Corrected erroneous 2024 date in prior changelog entries
|
||||
|
||||
## [5.7.1-beta.1] 2025-04-17
|
||||
## Changed
|
||||
- Updated geolocation records
|
||||
|
||||
## [5.7.0-beta.13] 2025-04-17
|
||||
## Fixed
|
||||
- Updated checksum for UI helper file
|
||||
|
||||
## [5.7.0-beta.12] 2024-04-04
|
||||
## [5.7.0-beta.12] 2025-04-04
|
||||
## Dropped
|
||||
- Removed extraneous pre-boot API checks that could cause error messages to be printed if the user had not set up an API key yet
|
||||
|
||||
## [5.7.0-beta.11] 2024-03-20
|
||||
## [5.7.0-beta.11] 2025-03-20
|
||||
## Fixed
|
||||
- Reduce startup time when testing remote APIs
|
||||
|
||||
## [5.7.0-beta.10] 2024-03-15
|
||||
## [5.7.0-beta.10] 2025-03-15
|
||||
## Added
|
||||
- Restore application size on subsequent startup
|
||||
## Fixed
|
||||
|
|
@ -23,44 +33,44 @@
|
|||
- Packed filter checkbox buttons into a 3x3 grid
|
||||
- Updated IP database to 2025-03 records
|
||||
|
||||
## [5.7.0-beta.9] 2024-03-04
|
||||
## [5.7.0-beta.9] 2025-03-04
|
||||
## Fixed
|
||||
- Livonia server results being dropped from batch queries
|
||||
|
||||
## [5.7.0-beta.8] 2024-02-10
|
||||
## [5.7.0-beta.8] 2025-02-10
|
||||
## Changed
|
||||
- Drop launch flag and check for invocation through Steam automatically
|
||||
|
||||
## [5.7.0-beta.7] 2024-02-09
|
||||
## [5.7.0-beta.7] 2025-02-09
|
||||
## Changed
|
||||
- Update IP database records for 2025-02
|
||||
|
||||
## [5.7.0-beta.6] 2024-01-10
|
||||
## [5.7.0-beta.6] 2025-01-10
|
||||
## Fixed
|
||||
- Resolve regression introduced with IP resolution feature in 13c6813
|
||||
|
||||
## [5.7.0-beta.5] 2024-01-10
|
||||
## [5.7.0-beta.5] 2025-01-10
|
||||
## Changed
|
||||
- Stricter redacting of usernames (again)
|
||||
|
||||
## [5.7.0-beta.4] 2024-01-10
|
||||
## [5.7.0-beta.4] 2025-01-10
|
||||
## Fixed
|
||||
- Clerical hotfix
|
||||
|
||||
## [5.7.0-beta.3] 2024-01-10
|
||||
## [5.7.0-beta.3] 2025-01-10
|
||||
## Changed
|
||||
- Support legacy jq syntax for Ubuntu variants
|
||||
|
||||
## [5.7.0-beta.2] 2024-01-07
|
||||
## [5.7.0-beta.2] 2025-01-07
|
||||
## Fixed
|
||||
- Stricter redacting of usernames
|
||||
- Omit nested directories when traversing symlinks
|
||||
|
||||
## [5.7.0-beta.1] 2024-01-07
|
||||
## [5.7.0-beta.1] 2025-01-07
|
||||
### Changed
|
||||
- Normalize version number
|
||||
|
||||
## [5.6.0-beta.21] 2024-01-06
|
||||
## [5.6.0-beta.21] 2025-01-06
|
||||
### Added
|
||||
- Add in-app documentation link to Codeberg mirror
|
||||
- Hover tooltips to most buttons
|
||||
|
|
|
|||
130
dzgui.sh
130
dzgui.sh
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env bash
|
||||
set -o pipefail
|
||||
|
||||
version=5.7.1-beta.1
|
||||
version=5.8.1-beta.1
|
||||
|
||||
#CONSTANTS
|
||||
aid=221100
|
||||
|
|
@ -49,10 +49,9 @@ freedesktop_path="$HOME/.local/share/applications"
|
|||
|
||||
#HELPERS
|
||||
ui_helper="$helpers_path/ui.py"
|
||||
geo_file="$helpers_path/ips.csv"
|
||||
km_helper="$helpers_path/latlon"
|
||||
sums_path="$helpers_path/sums.md5"
|
||||
func_helper="$helpers_path/funcs"
|
||||
geo_helper="$helpers_path/ips.csv"
|
||||
|
||||
#REMOTE
|
||||
remote_host=gh
|
||||
|
|
@ -63,7 +62,6 @@ stable_url="$url_prefix/dzgui"
|
|||
testing_url="$url_prefix/testing"
|
||||
releases_url="https://github.com/$author/$repo/releases/download/browser"
|
||||
km_helper_url="$releases_url/latlon"
|
||||
geo_file_url="$releases_url/ips.csv.gz"
|
||||
|
||||
|
||||
set_im_module(){
|
||||
|
|
@ -637,18 +635,8 @@ fetch_helpers_by_sum(){
|
|||
done
|
||||
return 0
|
||||
}
|
||||
fetch_geo_file(){
|
||||
# for binary releases
|
||||
local geo_sum="83bf7655a4197214b4be73dfa9a7b319"
|
||||
fetch_km_helper(){
|
||||
local km_sum="b038fdb8f655798207bd28de3a004706"
|
||||
local gzip="$helpers_path/ips.csv.gz"
|
||||
if [[ ! -f $geo_file ]] || [[ $(get_hash $geo_file) != $geo_sum ]]; then
|
||||
local res=$(get_response_code "$geo_file_url")
|
||||
[[ $res -ne 200 ]] && raise_error_and_quit "Remote resource unavailable: '$geo_file_url'"
|
||||
curl -Ls "$geo_file_url" > "$gzip"
|
||||
#force overwrite
|
||||
gunzip -f "$gzip"
|
||||
fi
|
||||
if [[ ! -f $km_helper ]] || [[ $(get_hash $km_helper) != $km_sum ]]; then
|
||||
local res=$(get_response_code "$km_helper_url")
|
||||
[[ $res -ne 200 ]] && raise_error_and_quit "Remote resource unavailable: '$km_helper_url'"
|
||||
|
|
@ -656,10 +644,119 @@ fetch_geo_file(){
|
|||
chmod +x "$km_helper"
|
||||
fi
|
||||
}
|
||||
get_response_code(){
|
||||
local url="$1"
|
||||
curl -Ls -I -o /dev/null -w "%{http_code}" "$url"
|
||||
}
|
||||
raise_error_and_quit(){
|
||||
echo "$1"
|
||||
exit 1
|
||||
}
|
||||
fetch_ip_db(){
|
||||
parse_dl_url(){
|
||||
curl -Ls "$url" \
|
||||
| grep "csv.gz" \
|
||||
| awk -F"['']" '{print $2}'
|
||||
}
|
||||
|
||||
parse_dl_url_date(){
|
||||
local url="$1"
|
||||
<<< "$url" \
|
||||
awk -F/ '{print $5}' \
|
||||
| awk -F"dbip-city-lite-" '{print $2}' \
|
||||
| awk -F'.csv.gz' '{print $1}'
|
||||
}
|
||||
|
||||
fetch(){
|
||||
logger INFO "Triggering fetch routine"
|
||||
local url="$1"
|
||||
|
||||
curl -Ls "$url" > "$base_file"
|
||||
if [[ $? -ne 0 ]]; then
|
||||
logger WARN "Abnormal exit while accessing '$url'"
|
||||
return
|
||||
fi
|
||||
|
||||
gunzip -f "$base_file"
|
||||
if [[ $? -ne 0 ]]; then
|
||||
rm "$base_file"
|
||||
logger WARN "Abnormal exit while unpacking gzip '$base_file'"
|
||||
return
|
||||
fi
|
||||
|
||||
< "$extracted_file" grep -vE "^[a-z0-9]{4}:" | grep -v "::" > "$ip_file"
|
||||
if [[ $? -ne 0 ]]; then
|
||||
logger WARN "Abnormal exit while parsing IPs"
|
||||
return
|
||||
fi
|
||||
rm "${state_path}/${month}.csv"
|
||||
|
||||
readarray -t records < <(cat $ip_file | awk -F, 'NR==1 {print $1} END {print $1}')
|
||||
if [[ ${records[0]} != "0.0.0.0" ]] && [[ ${records[1]} != "224.0.0.0" ]]; then
|
||||
logger WARN "Anchor records missing in '$ip_file', possibly malformed"
|
||||
rm "$ip_file"
|
||||
return
|
||||
fi
|
||||
|
||||
mv "$ip_file" "$geo_helper"
|
||||
if [[ $? -ne 0 ]]; then
|
||||
logger WARN "Abnormal exit while moving '$ip_file' to '$helpers_path'"
|
||||
rm "$ip_file"
|
||||
fi
|
||||
|
||||
echo "$month" > "$month_file"
|
||||
logger INFO "Wrote '$month' to stub '$month_file'"
|
||||
logger INFO "Updated '$ip_file'"
|
||||
}
|
||||
|
||||
local url="https://db-ip.com/db/download/ip-to-city-lite"
|
||||
local month_file="${state_path}/.month"
|
||||
local ip_file="${state_path}/ips.csv"
|
||||
|
||||
# test main url
|
||||
local res=$(get_response_code "$url")
|
||||
if [[ $res -ne 200 ]]; then
|
||||
logger WARN "Failed to retrieve remote resource: '$url' ($res)"
|
||||
return
|
||||
fi
|
||||
logger INFO "Resolved remote URL: '$url'"
|
||||
|
||||
# test dl url
|
||||
local dl_url="$(parse_dl_url)"
|
||||
local res=$(get_response_code "$dl_url")
|
||||
if [[ $res -ne 200 ]]; then
|
||||
logger WARN "Remote resource unavailable: '$dl_url' ($res)"
|
||||
return
|
||||
fi
|
||||
logger INFO "Resolved download URL: '$dl_url'"
|
||||
|
||||
local month=$(parse_dl_url_date "$dl_url")
|
||||
local base_file="${state_path}/${month}.csv.gz"
|
||||
local extracted_file="${state_path}/${month}.csv"
|
||||
|
||||
# no stub file
|
||||
if [[ ! -f $month_file ]]; then
|
||||
logger WARN "No stub file '$month_file' present"
|
||||
fetch "$dl_url"
|
||||
return
|
||||
fi
|
||||
|
||||
# local needs update
|
||||
local last_month=$(< "$month_file")
|
||||
if [[ $last_month != "$month" ]]; then
|
||||
logger WARN "Local stub '$last_month' does not match remote stub '$month'"
|
||||
fetch "$dl_url"
|
||||
return
|
||||
fi
|
||||
|
||||
# if stub is same date, abort
|
||||
logger INFO "Local stub '$last_month' is identical to remote, skipping"
|
||||
}
|
||||
fetch_helpers(){
|
||||
fetch_a2s
|
||||
fetch_dzq
|
||||
fetch_geo_file
|
||||
fetch_km_helper
|
||||
fetch_ip_db
|
||||
fetch_helpers_by_sum
|
||||
[[ ! -f $share_path/icon.png ]] && freedesktop_dirs
|
||||
fetch_icons
|
||||
|
|
@ -899,7 +996,6 @@ test_connection(){
|
|||
stable_url="$url_prefix/dzgui"
|
||||
testing_url="$url_prefix/testing"
|
||||
km_helper_url="$releases_url/latlon"
|
||||
geo_file_url="$releases_url/ips.csv.gz"
|
||||
fi
|
||||
}
|
||||
legacy_cols(){
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue