️️⚡️ SYNC ⚡️ 2024/11/26 07:24

.cron/Justfile

@@ -0,0 +1,15 @@
+root := `git rev-parse --show-toplevel`
+
+default: cron aggregate stats hallofshame
+
+cron:
+    pstats {{root}}/.cron/jobs/abuseipdb/cron
+
+aggregate:
+    pstats {{root}}/.cron/jobs/abuseipdb/aggregate
+
+stats:
+    pstats {{root}}/.cron/jobs/abuseipdb/stats
+
+hallofshame: stats
+    pstats {{root}}/.cron/jobs/abuseipdb/hallofshame

.cron/jobs/abuseipdb/aggregate

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Setup
+cd "$(dirname $0)"
+GIT_ROOT=$(git rev-parse --show-toplevel)
+
+. $GIT_ROOT/.cron/scripts/ciutil
+
+DB_PATH=$GIT_ROOT/db
+README_PATH=$GIT_ROOT/README.md
+
+DATE=$(date +%F)
+DATE_DIR=$DB_PATH/$DATE
+LATEST="$DATE_DIR/$DATE.ipv4"
+
+aggregate() {
+  local DAYS=$(expr $1 - 1)
+  local OUTPUT=$2
+  cd $DB_PATH
+  ___
+  echo "ℹ $OUTPUT"; echo
+
+  args=()
+
+  for i in $(seq 0 $DAYS); do
+    day=$(date +%Y-%m-%d -d "$(date) - $i days")
+    file=$day/$day.ipv4
+
+    if [[ -f $file ]]; then
+      args+=("$file")
+      # echo "- $(basename $file)"
+        echo "-  $file (`wc -l < $file` ip)"
+    else
+      echo "❌ $file does not exist"
+    fi
+
+  done
+
+  iprange "${args[@]}" --print-single-ips --except $GIT_ROOT/.cron/jobs/abuseipdb/bogons.ipv4 >| $GIT_ROOT/$OUTPUT.tmp
+
+  TS=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
+  echo "#" >| $GIT_ROOT/$OUTPUT
+  echo "#" >> $GIT_ROOT/$OUTPUT
+  echo "# Aggregated Blocklist for AbuseIPDB: A list of the most reported IP addresses." >> $GIT_ROOT/$OUTPUT
+  echo "#" >> $GIT_ROOT/$OUTPUT
+  echo "# Last updated:           $TS" >> $GIT_ROOT/$OUTPUT
+  echo "# Confidence level:       ~100%" >> $GIT_ROOT/$OUTPUT
+  echo "# Filename:               $OUTPUT" >> $GIT_ROOT/$OUTPUT
+  echo "# Number of ips:          $(wc -l < $GIT_ROOT/$OUTPUT.tmp)" >> $GIT_ROOT/$OUTPUT
+  echo "#" >> $GIT_ROOT/$OUTPUT
+  echo "# Source:                 https://github.com/borestad/blocklist-abuseipdb" >> $GIT_ROOT/$OUTPUT
+  echo "# Stats:                  https://github.com/borestad/blocklist-abuseipdb/tree/main/stats" >> $GIT_ROOT/$OUTPUT
+  echo "# Credits 1:              https://www.abuseipdb.com - please support them!" >> $GIT_ROOT/$OUTPUT
+  echo "# Credits 2:              https://ipinfo.io - The Trusted Source For IP Address Data" >> $GIT_ROOT/$OUTPUT
+  echo "#" >> $GIT_ROOT/$OUTPUT
+  echo "#" >> $GIT_ROOT/$OUTPUT
+
+  cat $GIT_ROOT/$OUTPUT.tmp >> $GIT_ROOT/$OUTPUT
+  echo
+  echo "Total: (`wc -l < $GIT_ROOT/$OUTPUT.tmp` ip)"
+
+  rm -f $GIT_ROOT/$OUTPUT.tmp
+}
+
+
+decorate-with-asn-info() {
+  cd $GIT_ROOT && \
+  fd -tf '.ipv4$' --max-depth=1 --min-depth=1 -x \
+    bash -c "cat {} | .cron/scripts/ip2ipinfo.ts >| {}.tmp && mv {}.tmp {}" || true
+}
+
+update-footer() {
+  echo "✨ Update footer"
+
+  # Delete everything below placeholder
+  sed -i '/ABUSEIPDB-STATS-PLACEHOLDER/q' $README_PATH
+
+  update=$(date -u '+%Y-%m-%d - %H:%M:%S')
+  echo "Last check: \`$update\` (UTC)" >> $README_PATH
+
+  echo '```' >> $README_PATH
+
+  cd $GIT_ROOT && find . -mindepth 1 -maxdepth 1 -iname 'abuseipdb-s100*.ipv4' -print0 | sort -zV | xargs -I {} -0 sh -c 'name=$(basename {}); echo "$name ($(wc -l < $name) ip)"' >> $README_PATH
+  echo '```' >> $README_PATH
+}
+
+fd '\.ipv4$' $DB_PATH -x cat | iprange - --print-single-ips --except $GIT_ROOT/.cron/jobs/abuseipdb/bogons.ipv4 | sponge $GIT_ROOT/abuseipdb-s100-all.ipv4 &
+# iprange $DB_PATH/**/*.ipv4 --print-single-ips | sponge $GIT_ROOT/abuseipdb-s100-all.ipv4 &
+
+# c = confidence
+aggregate 2   "abuseipdb-s100-1d.ipv4" # Compensate +24h to ensure we have a full day of data
+aggregate 3   "abuseipdb-s100-3d.ipv4"
+aggregate 7   "abuseipdb-s100-7d.ipv4"
+aggregate 14  "abuseipdb-s100-14d.ipv4"
+aggregate 30  "abuseipdb-s100-30d.ipv4"
+aggregate 60  "abuseipdb-s100-60d.ipv4"
+aggregate 90  "abuseipdb-s100-90d.ipv4"
+aggregate 120  "abuseipdb-s100-120d.ipv4"
+
+wait
+
+update-footer
+
+decorate-with-asn-info

.cron/jobs/abuseipdb/asn

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -e
+export LC_ALL=C
+
+# Setup
+cd "$(dirname $0)"
+GIT_ROOT=$(git rev-parse --show-toplevel)
+
+cat $GIT_ROOT/.cron/jobs/abuseipdb/asn.cfg | shfmt -mn | \
+  xargs -P2 -I% bkt --ttl=1d -- curl -sL https://raw.githubusercontent.com/ipverse/asn-ip/master/as/%/ipv4-aggregated.txt | \
+  iprange --min-prefix 24

.cron/jobs/abuseipdb/asn.cfg

@@ -0,0 +1,9 @@
+# These ASNS are broken. Relative to their assigned IP space, they have an extremely high number of malicious traffic.
+# Continously check for maliscoius activity, even if Confidence Level < 100%
+
+215766   # AS215766 (EMANUELHOSTING) Emanuel Hosting Ltd. - United Kingdom https://www.abuseipdb.com/check-block/79.110.62.0/24
+201814   # AS201814 (MEVSPACE) MEVSPACE sp. z o.o. - Poland
+267784   # AS267784 (FLYSERVERS) Flyservers S.A https://www.abuseipdb.com/check-block/45.43.64.0/24 - Panama
+211298   # AS211298 (INTERNET-MEASUREMENT) Constantine Cybersecurity Ltd. - United Kingdom https://www.abuseipdb.com/check-block/87.236.176.0/24
+202425   # AS202425 (INT-NETWORK) IP Volume inc https://cleantalk.org/blacklists/as202425 - United Kingdom
+208843   # AS208843 (ALPHASTRIKE-RESEARCH) Alpha Strike Labs GmbH - Germany https://www.abuseipdb.com/check-block/45.83.67.0/24

.cron/jobs/abuseipdb/bogons.ipv4

@@ -0,0 +1,13 @@
+0.0.0.0/8
+10.0.0.0/8
+100.64.0.0/10
+127.0.0.0/8
+169.254.0.0/16
+172.16.0.0/12
+192.0.0.0/24
+192.0.2.0/24
+192.168.0.0/16
+198.18.0.0/15
+198.51.100.0/24
+203.0.113.0/24
+224.0.0.0/3

.cron/jobs/abuseipdb/cron

@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Setup
+cd "$(dirname $0)"
+
+GIT_ROOT=$(git rev-parse --show-toplevel)
+DB_PATH=$GIT_ROOT/db
+mkdir -p $DB_PATH
+TEMPFILE=$(mktemp)
+TEMPDIR=$(mktemp -d)
+
+cd $TEMPDIR
+
+# Debug
+echo "Public IP:"
+echo $(timeout 2s curl --no-progress-meter ipv4.icanhazip.com)
+echo
+
+echo '✔ Debug...'
+date '+%Y/%m/%d %H:%M:%S'
+bkt --ttl=6h -- date '+%Y/%m/%d %H:%M:%S'
+echo
+
+
+echo '✔ Download abuseipdb...'
+# Use a TTL of ~2.5 hours (~ 9/10 requests if verified webmaster)
+bkt --ttl=180min -- curl https://api.abuseipdb.com/api/v2/blacklist \
+  --get \
+  --max-time 10 \
+  --user-agent "" \
+  --no-progress-meter \
+  -d confidenceMinimum=100 \
+  -d limit=9999999 \
+  -H "Key: $ABUSEIPDB_TOKEN" \
+  -H "Accept: text/plain" \
+  --fail \
+  -w "\n" \
+  -o TEMPFILE.1 || true
+
+echo '✔ Download & decorate with extra sources ...'
+echo '#2: abuseipdb.tmiland.com/'
+curl -sL https://abuseipdb.tmiland.com/abuseipdb.txt \
+  --compressed --max-time 10 -G -sL --fail -o TEMPFILE.2 || true
+
+echo '#3: LittleJake'
+curl -sL https://raw.githubusercontent.com/LittleJake/ip-blacklist/main/abuseipdb_blacklist_ip_score_100.txt \
+  --compressed --max-time 10 -G -sL --fail -o TEMPFILE.3 || true
+
+# 💩 Whitelisted scanner: Palo Alto
+echo '#4: Palo Alto'
+curl -sL https://raw.githubusercontent.com/borestad/iplists/refs/heads/main/paloaltonetworks/paloaltonetworks.ipv4 | \
+  grepip | xargs -n1 $GIT_ROOT/.cron/scripts/abuseipdb-check | jq 'select(.numReports > 1)' | grepip >| TEMPFILE.4 || true
+
+# 💩 Whitelisted scanner: Censys
+echo '#5: Censys'
+curl -sL https://raw.githubusercontent.com/borestad/iplists/refs/heads/main/censys/censys.ipv4 | \
+  grepip | xargs -n1 $GIT_ROOT/.cron/scripts/abuseipdb-check | jq 'select(.numReports > 1)' | grepip >| TEMPFILE.5 || true
+
+# 💩 Whitelisted scanner: Project Sonar (Rapid7)
+echo '#6: Project Sonar'
+curl -sL https://raw.githubusercontent.com/borestad/iplists/refs/heads/main/project-sonar/project-sonar.ipv4 | \
+  grepip | xargs -n1 $GIT_ROOT/.cron/scripts/abuseipdb-check | jq 'select(.numReports > 1)' | grepip >| TEMPFILE.6 || true
+
+echo '#7: Broken ASNS'
+$GIT_ROOT/.cron/jobs/abuseipdb/asn | \
+  xargs -I% $GIT_ROOT/.cron/scripts/abuseipdb-check % | jq 'select(.numReports > 1)' | grepip >| TEMPFILE.7 || true
+
+# Redundancy:
+# - Separate private cache (1 of 5 requests / day) to avoid breaking the 5 free run limit / day
+# - If above urls fail due to github actions being flaky, still have somewhat fresh data.
+# echo '✔ Download from cache'
+echo '#8: Private cache'
+curl "$CRONSRC_URL" --compressed --max-time 10 -G -sL -w "\n\n" --fail -o TEMPFILE.8 || true
+
+# echo '✔ Stats'
+# for FILE in TEMPFILE.*; do printf "$FILE "; wc -l < $FILE; done
+
+echo '✔ Squash all sources (by design: fail if no sources worked)'
+grep -h "" TEMPFILE.* >> $TEMPFILE
+
+echo '✔ Validate: Clean comments'
+cat $TEMPFILE | shfmt -mn | sponge $TEMPFILE
+
+echo '✔ Validate: Extract ipv6 data'
+  grep ':' $TEMPFILE | sort | tac | cidr-merger | sponge $TEMPFILE.ipv6
+
+echo '✔ Validate: Extract ipv4 data'
+  grep -v ":" $TEMPFILE | \
+  iprange - --print-single-ips --except $GIT_ROOT/.cron/jobs/abuseipdb/bogons.ipv4 \
+  > $TEMPFILE.ipv4
+
+# 3. Validate data
+LINES=`wc -l < $TEMPFILE.ipv4`
+if [[ "$LINES" -gt "1000" ]]; then
+  echo "✔ Validate: File contains: $LINES lines"
+  mv $TEMPFILE.ipv4 $DB_PATH/abuseipdb-s100-latest.ipv4
+  mv $TEMPFILE.ipv6 $DB_PATH/abuseipdb-s100-latest.ipv6
+else
+  echo "❌ Validation failed"
+  echo
+  echo "-----------------------------------------------------"
+  cat $TEMPFILE
+  echo "-----------------------------------------------------"
+  cat $TEMPFILE.ipv4
+  echo "-----------------------------------------------------"
+  exit 1
+fi
+
+echo
+echo '✔ Aggregate: Create folders'
+DATE=$(date +%F)
+DATE_DIR=$DB_PATH/$DATE
+mkdir -pv $DATE_DIR && cd $DATE_DIR
+
+echo '✔ Aggregate: Copy latest to correct date folder'
+cp $DB_PATH/abuseipdb-s100-latest.ipv4 "$DATE_DIR/tmp-$(date +%H-%m-%S).ipv4"
+cp $DB_PATH/abuseipdb-s100-latest.ipv6 "$DATE_DIR/tmp-$(date +%H-%m-%S).ipv6"
+
+echo '✔ Aggregate: Squash ipv4 data'
+iprange --print-single-ips *.ipv4 | sponge $(date +%Y-%m-%d).ipv4
+
+echo '✔ Aggregate: Squash ipv6 data'
+cat *.ipv6 | grep ':' | sort | uniq | sort | sponge $(date +%Y-%m-%d).ipv6
+
+echo
+echo '✔ Cleanup: Remove temp files'
+rm -f tmp*.ipv4
+rm -f tmp*.ipv6

.cron/jobs/abuseipdb/hallofshame

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+set -e
+export LC_ALL=C
+
+# Setup
+cd "$(dirname $0)"
+GIT_ROOT=$(git rev-parse --show-toplevel)
+mkdir -p $GIT_ROOT/stats/hallofshame/subnets
+
+cd $GIT_ROOT
+
+update-hall-of-shame() {
+  local days=$1
+  local percent=$2
+  local nr=$(($percent * 256 / 100))
+
+  OUTPUT="$GIT_ROOT/stats/hallofshame/subnets/abuseipdb-s99-hallofshame-${days}-${percent}percent.ipv4"
+  echo $OUTPUT
+  TS=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
+  TEMPFILE=$(mktemp)
+
+
+  # Hall of shame, where > x % of the ips in a cidr-block is malicious traffic
+  cat "./abuseipdb-s100-${days}.ipv4" | \
+    iprange -1 | \
+    sed 's/\./ /g' | \
+    awk '{print $1, $2, $3}' | \
+    sed 's/[[:space:]]/./g' | \
+    sort | uniq -c | sort | \
+    sed 's/$/.0\/24/' | \
+    awk "\$1 >= $nr {print \$2}" | \
+    iprange --min-prefix 24 \
+    >> $TEMPFILE
+
+
+  echo "#" >| $OUTPUT
+  echo "# Hall of Shame: $days" >> $OUTPUT
+  echo "# An aggregated list of ip ranges, where more than ${percent}% the traffic from a /24 range is malicious from the last ${days}ays" >> $OUTPUT
+  echo "#" >> $OUTPUT
+  echo "#" >> $OUTPUT
+  echo "# Last updated:        $TS" >> $OUTPUT
+  echo "# Days:                $days" >> $OUTPUT
+  echo "# Malicious level:     > ${percent}%" >> $OUTPUT
+  echo "# Filename:            $(basename $OUTPUT)" >> $OUTPUT
+  echo "# Number of ips:       $(iprange -1 $TEMPFILE | wc -l)" >> $OUTPUT
+  echo "#" >> $OUTPUT
+  echo "# Source:              https://github.com/borestad/blocklist-abuseipdb" >> $OUTPUT
+  echo "# Credits:             https://www.abuseipdb.com - please support them!" >> $OUTPUT
+  echo "# Example:             https://www.abuseipdb.com/check-block/64.62.156.0/24" >> $OUTPUT
+
+  echo "#" >> $OUTPUT
+  echo "" >> $OUTPUT
+
+  cat $TEMPFILE >> $OUTPUT
+
+}
+
+decorate-with-asn-info() {
+  cd $GIT_ROOT && \
+  fd -tf '.ipv4$' $GIT_ROOT/stats/hallofshame/subnets --max-depth=1 --min-depth=1 -x \
+    bash -c "cat {} | .cron/scripts/ip2ipinfo.ts >| {}.tmp && mv {}.tmp {}" || true
+}
+
+
+
+for i in 5 25 50 75; do
+  update-hall-of-shame 1d $i &
+  update-hall-of-shame 30d $i &
+  update-hall-of-shame all $i &
+done
+
+wait
+
+decorate-with-asn-info
+