v0.6.1

.gitlab-ci.yml

@@ -25,9 +25,11 @@ build:
   - "apk add sqlite-static"
   - "shards install"
   - "crystal build --release --static src/access_log.cr"
-  - "strip --strip-all access_log"
+  - "crystal build --release --static src/ip2asn_server.cr"
+  - "strip --strip-all access_log ip2asn_server"
   - "install -Dm 644 LICENSE ${RELEASE_DIRECTORY}/LICENSE"
   - "install -Dm 755 access_log ${RELEASE_DIRECTORY}/access_log"
+  - "install -Dm 755 ip2asn_server ${RELEASE_DIRECTORY}/ip2asn_server"
   - "tar -cf ${RELEASE_TARBALL} ${RELEASE_DIRECTORY}"
   artifacts:
     paths:

contrib/asn_database.sh

@@ -3,6 +3,8 @@ set -e
 
 db="${1:-$PWD/asn.sqlite3}"
 sqlite3 "$db" "CREATE TABLE IF NOT EXISTS asn (range_start INTEGER, range_end INTEGER, id INTEGER, country STRING, description TEXT);"
+sqlite3 "$db" "CREATE INDEX IF NOT EXISTS asn_id_idx ON asn (id);"
+sqlite3 "$db" "CREATE TABLE IF NOT EXISTS status (id INTEGER PRIMARY KEY, blocked BOOLEAN DEFAULT FALSE);"
 sqlite3 "$db" "DELETE from asn;"
 wget https://iptoasn.com/data/ip2asn-v4-u32.tsv.gz -O - | gunzip | sqlite3 -tabs "$db" ".import '|cat -' asn"
 

contrib/create.sql

@@ -55,7 +55,8 @@ CREATE TABLE IF NOT EXISTS "access_logs" (
  "production_co2" float DEFAULT NULL,
  "total_co2" float DEFAULT NULL,
  "node" varchar DEFAULT NULL,
- "asn" integer DEFAULT NULL
+ "asn" integer DEFAULT NULL,
+ "http3" varchar(2) DEFAULT NULL
 );
 
 CREATE INDEX IF NOT EXISTS "index_access_logs_on_host" ON "access_logs" ("host");

contrib/ip2asn_nginx.conf

@@ -0,0 +1,19 @@
+keyval_zone_redis zone=ip2asn hostname=::1;
+keyval "$remote_addr:id" $asn zone=ip2asn;
+keyval "$remote_addr:blocked" $asn_blocked zone=ip2asn;
+
+limit_conn_zone $asn zone=asn_conn:10m;
+limit_req_zone $asn zone=asn_req:10m rate=10r/s;
+
+server {
+  server_name _;
+
+  listen 80 default_server;
+
+  if ($asn_blocked) {
+    return 444;
+  }
+
+  limit_conn asn_conn 1;
+  limit_req zone=asn_req nodelay;
+}

shard.yml

@@ -1,5 +1,5 @@
 name: "access_log"
-version: "0.6.0"
+version: "0.6.1"
 authors:
 - "f <f@sutty.nl>"
 targets:

src/access_log.cr

@@ -12,7 +12,7 @@ require "./models/crawler"
 require "./swd"
 require "./asn"
 
-VERSION = "0.6.0"
+VERSION = "0.6.1"
 
 Log.setup_from_env
 
@@ -28,7 +28,7 @@ crawler = false
 crawlers = [] of Crawler
 # Fields in database
 # TODO: Obtain them from AccessLog
-fields = %w[id remote_user host msec server_protocol request_method request_completion uri request_uri query_string status sent_http_content_type sent_http_content_encoding sent_http_etag sent_http_last_modified http_accept http_accept_encoding http_accept_language http_pragma http_cache_control http_if_none_match http_dnt http_user_agent http_origin http_referer request_time bytes_sent body_bytes_sent request_length http_connection pipe connection_requests geoip2_data_country_name geoip2_data_city_name ssl_server_name ssl_protocol ssl_early_data ssl_session_reused ssl_curves ssl_ciphers ssl_cipher sent_http_x_xss_protection sent_http_x_frame_options sent_http_x_content_type_options sent_http_strict_transport_security nginx_version pid crawler datacenter_co2 network_co2 consumer_device_co2 production_co2 total_co2 node asn]
+fields = %w[id remote_user host msec server_protocol request_method request_completion uri request_uri query_string status sent_http_content_type sent_http_content_encoding sent_http_etag sent_http_last_modified http_accept http_accept_encoding http_accept_language http_pragma http_cache_control http_if_none_match http_dnt http_user_agent http_origin http_referer request_time bytes_sent body_bytes_sent request_length http_connection pipe connection_requests geoip2_data_country_name geoip2_data_city_name ssl_server_name ssl_protocol ssl_early_data ssl_session_reused ssl_curves ssl_ciphers ssl_cipher sent_http_x_xss_protection sent_http_x_frame_options sent_http_x_content_type_options sent_http_strict_transport_security nginx_version pid crawler datacenter_co2 network_co2 consumer_device_co2 production_co2 total_co2 node asn http3]
 # Params for the query
 params = [] of String
 # SWD
@@ -220,7 +220,8 @@ DB.open database do |db|
               (swd ? s.try(&.production_co2) : nil),
               (swd ? s.try(&.total_co2) : nil),
               (node ? System.hostname : nil),
-              a.try(&.id)
+              a.try(&.id),
+              access_log.http3
 
     # Ignore parsing errors
     rescue e : JSON::ParseException

src/asn.cr

@@ -7,10 +7,12 @@ class AsnResult
   property id : Int64?
   property country : String?
   property description : String?
+  property blocked : Bool?
 end
 
 class ASN
   QUERY = "select id, country, description from asn where ? between range_start and range_end limit 1;"
+  STATUS = "select blocked from status where id = ? limit 1;"
 
   property db : DB::Database
 
@@ -30,7 +32,9 @@ class ASN
   end
 
   def query(binary_address : UInt32) : AsnResult?
-    @cache[binary_address] ||= @db.query_one(QUERY, binary_address.to_i64, &.read(AsnResult))
+    (@cache[binary_address] ||= @db.query_one(QUERY, binary_address.to_i64, &.read(AsnResult))).tap do |result|
+      result.blocked = @db.query_one(STATUS, result.id, &.read(Bool))
+    end
   rescue DB::NoResultsError
     nil
   end

src/ip2asn_server.cr

@@ -0,0 +1,90 @@
+# This is a very minimal Redis server that works with Nginx keyval
+# module.  When set like this:
+#
+# keyval_zone_redis zone=redis hostname=::1;
+# keyval "$remote_addr:id" $asn_id zone=redis;
+#
+# We can query the ASN id from a database and use it on Nginx config to
+# rate limit entire subnets.
+require "log"
+require "option_parser"
+require "./asn"
+require "./redis_server"
+
+Log.setup_from_env
+
+VERSION = "0.2.0"
+ASN_ID = "id"
+ASN_DESCRIPTION = "description"
+ASN_COUNTRY = "country"
+ASN_BLOCKED = "blocked"
+
+asn_database = "sqlite3://./asn.sqlite3"
+interface = "localhost"
+port = 6379
+
+OptionParser.parse do |p|
+  p.banner = "IP to ASN Redis server"
+
+  p.on "-v", "--version", "Show version" do
+    puts VERSION
+    exit
+  end
+
+  p.on "-h", "--help", "Show help" do
+    puts p
+    exit
+  end
+
+  p.on "-d #{asn_database}", "--asn-database=#{asn_database}", "ASN Database URI" do |d|
+    asn_database = d
+  end
+
+  p.on "-i #{interface}", "--interface=#{interface}", "Listening interface" do |i|
+    interface = i
+  end
+
+  p.on "-p #{port}", "--port=#{port}", "Listening port" do |p|
+    port = p.to_i
+  end
+end
+
+asn = ASN.new(asn_database)
+
+# The client can be a long lived connection sending commands. We only
+# answer to GET commands.
+RedisServer.new(interface, port) do |client, _, key|
+  begin
+    _, ip, field = key.split(":", 3)
+  rescue
+    Log.warn &.emit("Error parsing key", key: key)
+    next
+  end
+
+  # This is cached in memory
+  result = asn.query(ip)
+
+  next if result.nil?
+
+  # XXX: Are we going to use the other fields?
+  response =
+    case field
+    when ASN_ID then result.try(&.id).try(&.to_s)
+    when ASN_COUNTRY then result.try(&.country)
+    when ASN_DESCRIPTION then result.try(&.description)
+    when ASN_BLOCKED then
+      if result.try(&.blocked)
+        "1"
+      else
+        "0"
+      end
+    else nil
+    end
+
+  next if response.nil?
+
+  # Reply
+  RedisServer.bulk_string(client, response)
+end
+
+asn.close

src/models/access_log.cr

@@ -59,4 +59,5 @@ class AccessLog
   property nginx_version : String
   property pid : String
   property remote_addr : String?
+  property http3 : String?
 end

src/redis_server.cr

@@ -0,0 +1,59 @@
+require "log"
+require "socket"
+
+class RedisServer
+  BULK_STRING = "$"
+  CRLF = "\r\n"
+  NOT_IMPLEMENTED = "-Not implemented#{CRLF}"
+  QUIT_COMMAND = "quit"
+  GET_COMMAND = "get"
+
+  property server : TCPServer
+
+  def initialize(interface : String = "localhost", port : Int = 6379, &block : TCPSocket, String, String -> _)
+    @server = TCPServer.new(interface, port)
+
+    while (client = server.accept?)
+      spawn handle_client(client, block)
+    end
+  ensure
+    server.close
+  end
+
+  def handle_client(client : TCPSocket, block : Proc)
+    next_is_key = false
+    last_command = ""
+
+    while (line = client.gets)
+      # We're not a compliant RESP
+      next if line.starts_with?("*")
+      next if line.starts_with?("$")
+
+      # Execute the query when we get the key
+      if next_is_key
+        next_is_key = false
+
+        # Do something with the key
+        block.call(client, last_command, line)
+      # First we get the Redis command and we only support a small subset.
+      else
+        case (last_command = line.downcase)
+        when QUIT_COMMAND then client.close
+        when GET_COMMAND then next_is_key = true
+        else
+          client.print NOT_IMPLEMENTED
+
+          raise last_command
+        end
+      end
+    end
+  rescue ex
+    Log.warn &.emit("Error", exception: ex.class.name, error: ex.message)
+  ensure
+    client.close
+  end
+
+  def self.bulk_string(client : TCPSocket, string : String)
+    client.print(BULK_STRING, string.size, CRLF, string, CRLF)
+  end
+end

src/swd/intensity_data.cr

@@ -157,7 +157,7 @@ class SWD
       "PM": 600.0,
       "PR": 664.53,
       "PS": 460.78,
-      "PT": 112.29,
+      "PT": 111.8,
       "PY": 24.86,
       "QA": 602.83,
       "RE": 525.22,
@@ -169,9 +169,9 @@ class SWD
       "SB": 636.36,
       "SC": 571.43,
       "SD": 214.33,
-      "SE": 35.82,
+      "SE": 35.89,
       "SG": 498.74,
-      "SI": 227.65,
+      "SI": 227.11,
       "SK": 96.49,
       "SL": 47.62,
       "SN": 535.4,
@@ -208,7 +208,7 @@ class SWD
       "WS": 400.0,
       "XK": 958.72,
       "YE": 586.32,
-      "ZA": 713.48,
+      "ZA": 713.9,
       "ZM": 111.0,
       "ZW": 298.44
     }