v0.6.2

contrib/asn.sql

@@ -0,0 +1,12 @@
+BEGIN;
+CREATE TABLE IF NOT EXISTS asn_import (range_start INTEGER, range_end INTEGER, id INTEGER, country STRING, description TEXT);
+CREATE TABLE IF NOT EXISTS asns (range_start INTEGER, range_end INTEGER, id INTEGER, country STRING, owner_id INTEGER);
+CREATE UNIQUE INDEX IF NOT EXISTS asn_import_range_id ON asn_import (range_start, range_end);
+CREATE UNIQUE INDEX IF NOT EXISTS asns_range_id ON asns (range_start, range_end);
+CREATE INDEX IF NOT EXISTS asns_id_idx ON asns (id);
+CREATE INDEX IF NOT EXISTS asns_owner_id_idx ON asns (owner_id);
+CREATE TABLE IF NOT EXISTS owners (id INTEGER PRIMARY KEY, description TEXT);
+CREATE TABLE IF NOT EXISTS status (id INTEGER PRIMARY KEY, blocked BOOLEAN DEFAULT FALSE);
+CREATE TABLE IF NOT EXISTS owner_status (id INTEGER PRIMARY KEY, blocked BOOLEAN DEFAULT FALSE);
+DELETE FROM asn_import;
+COMMIT;

contrib/asn_2.sql

@@ -0,0 +1,12 @@
+BEGIN;
+INSERT INTO owners (description) SELECT DISTINCT asn_import.description FROM asn_import LEFT JOIN owners ON asn_import.description = owners.description WHERE owners.description IS NULL;
+INSERT INTO asns (range_start, range_end, id, country, owner_id)
+  SELECT asn_import.range_start, asn_import.range_end, asn_import.id, asn_import.country, owners.id
+  FROM asn_import
+  LEFT JOIN owners
+  ON owners.description = asn_import.description
+ON CONFLICT DO UPDATE SET id = excluded.id, country = excluded.country, owner_id = excluded.owner_id;
+DROP TABLE asn_import;
+DROP INDEX asns_range_id;
+COMMIT;
+VACUUM;

contrib/asn_database.sh

@@ -1,11 +1,10 @@
 #!/bin/sh
 set -e
 
+cr="$(dirname "$(readlink -f "$0")")"
 db="${1:-$PWD/asn.sqlite3}"
-sqlite3 "$db" "CREATE TABLE IF NOT EXISTS asn (range_start INTEGER, range_end INTEGER, id INTEGER, country STRING, description TEXT);"
-sqlite3 "$db" "CREATE INDEX IF NOT EXISTS asn_id_idx ON asn (id);"
-sqlite3 "$db" "CREATE TABLE IF NOT EXISTS status (id INTEGER PRIMARY KEY, blocked BOOLEAN DEFAULT FALSE);"
-sqlite3 "$db" "DELETE from asn;"
-wget https://iptoasn.com/data/ip2asn-v4-u32.tsv.gz -O - | gunzip | sqlite3 -tabs "$db" ".import '|cat -' asn"
+cat "$cr/asn.sql" | sqlite3 "$db"
+wget https://iptoasn.com/data/ip2asn-v4-u32.tsv.gz -O - | gunzip | sqlite3 -tabs "$db" ".import '|cat -' asn_import"
+cat "$cr/asn_2.sql" | sqlite3 "$db"
 
 echo "ASN database created/updated at $db"

contrib/create.sql

@@ -56,7 +56,12 @@ CREATE TABLE IF NOT EXISTS "access_logs" (
  "total_co2" float DEFAULT NULL,
  "node" varchar DEFAULT NULL,
  "asn" integer DEFAULT NULL,
- "http3" varchar(2) DEFAULT NULL
+ "http3" varchar(2) DEFAULT NULL,
+ "http_sec_fetch_mode" varchar DEFAULT NULL,
+ "http_sec_fetch_dest" varchar DEFAULT NULL,
+ "http_sec_fetch_site" varchar DEFAULT NULL,
+ "http_sec_fetch_user" varchar DEFAULT NULL,
+ "http_sec_purpose" varchar DEFAULT NULL
 );
 
 CREATE INDEX IF NOT EXISTS "index_access_logs_on_host" ON "access_logs" ("host");

shard.yml

@@ -1,5 +1,5 @@
 name: "access_log"
-version: "0.6.1"
+version: "0.6.2"
 authors:
 - "f <f@sutty.nl>"
 targets:

src/access_log.cr

@@ -12,7 +12,7 @@ require "./models/crawler"
 require "./swd"
 require "./asn"
 
-VERSION = "0.6.1"
+VERSION = "0.6.2"
 
 Log.setup_from_env
 
@@ -28,7 +28,7 @@ crawler = false
 crawlers = [] of Crawler
 # Fields in database
 # TODO: Obtain them from AccessLog
-fields = %w[id remote_user host msec server_protocol request_method request_completion uri request_uri query_string status sent_http_content_type sent_http_content_encoding sent_http_etag sent_http_last_modified http_accept http_accept_encoding http_accept_language http_pragma http_cache_control http_if_none_match http_dnt http_user_agent http_origin http_referer request_time bytes_sent body_bytes_sent request_length http_connection pipe connection_requests geoip2_data_country_name geoip2_data_city_name ssl_server_name ssl_protocol ssl_early_data ssl_session_reused ssl_curves ssl_ciphers ssl_cipher sent_http_x_xss_protection sent_http_x_frame_options sent_http_x_content_type_options sent_http_strict_transport_security nginx_version pid crawler datacenter_co2 network_co2 consumer_device_co2 production_co2 total_co2 node asn http3]
+fields = %w[id remote_user host msec server_protocol request_method request_completion uri request_uri query_string status sent_http_content_type sent_http_content_encoding sent_http_etag sent_http_last_modified http_accept http_accept_encoding http_accept_language http_pragma http_cache_control http_if_none_match http_dnt http_user_agent http_origin http_referer request_time bytes_sent body_bytes_sent request_length http_connection pipe connection_requests geoip2_data_country_name geoip2_data_city_name ssl_server_name ssl_protocol ssl_early_data ssl_session_reused ssl_curves ssl_ciphers ssl_cipher sent_http_x_xss_protection sent_http_x_frame_options sent_http_x_content_type_options sent_http_strict_transport_security nginx_version pid crawler datacenter_co2 network_co2 consumer_device_co2 production_co2 total_co2 node asn http3 http_sec_fetch_mode http_sec_fetch_dest http_sec_fetch_site http_sec_fetch_user http_sec_purpose]
 # Params for the query
 params = [] of String
 # SWD

src/asn.cr

@@ -6,13 +6,16 @@ class AsnResult
 
   property id : Int64?
   property country : String?
+  # TODO: Deprecate?
   property description : String?
+  property owner_id : Int64?
   property blocked : Bool?
 end
 
 class ASN
-  QUERY = "select id, country, description from asn where ? between range_start and range_end limit 1;"
+  QUERY = "select asns.id, country, owners.description, owner_id from asns left join owners on owners.id = asns.owner_id where ? between range_start and range_end limit 1;"
   STATUS = "select blocked from status where id = ? limit 1;"
+  OWNER_STATUS = "select blocked from owner_status where id = ? limit 1;"
 
   property db : DB::Database
 
@@ -33,7 +36,8 @@ class ASN
 
   def query(binary_address : UInt32) : AsnResult?
     (@cache[binary_address] ||= @db.query_one(QUERY, binary_address.to_i64, &.read(AsnResult))).tap do |result|
-      result.blocked = @db.query_one(STATUS, result.id, &.read(Bool))
+      result.blocked   = rescue_query { @db.query_one(STATUS, result.id, &.read(Bool)) }
+      result.blocked ||= rescue_query { @db.query_one(OWNER_STATUS, result.owner_id, &.read(Bool)) }
     end
   rescue DB::NoResultsError
     nil
@@ -42,4 +46,10 @@ class ASN
   def close
     @db.close
   end
+
+  def rescue_query(&)
+    yield
+  rescue DB::NoResultsError
+    nil
+  end
 end

src/ip2asn_server.cr

@@ -16,6 +16,7 @@ Log.setup_from_env
 VERSION = "0.2.0"
 ASN_ID = "id"
 ASN_DESCRIPTION = "description"
+ASN_OWNER_ID = "owner_id"
 ASN_COUNTRY = "country"
 ASN_BLOCKED = "blocked"
 
@@ -70,6 +71,7 @@ RedisServer.new(interface, port) do |client, _, key|
   response =
     case field
     when ASN_ID then result.try(&.id).try(&.to_s)
+    when ASN_OWNER_ID then result.try(&.owner_id).try(&.to_s)
     when ASN_COUNTRY then result.try(&.country)
     when ASN_DESCRIPTION then result.try(&.description)
     when ASN_BLOCKED then
@@ -81,10 +83,11 @@ RedisServer.new(interface, port) do |client, _, key|
     else nil
     end
 
-  next if response.nil?
-
-  # Reply
-  RedisServer.bulk_string(client, response)
+  if response.nil?
+    RedisServer.null(client)
+  else
+    RedisServer.bulk_string(client, response)
+  end
 end
 
 asn.close

src/models/access_log.cr

@@ -60,4 +60,9 @@ class AccessLog
   property pid : String
   property remote_addr : String?
   property http3 : String?
+  property http_sec_fetch_mode : String?
+  property http_sec_fetch_dest : String?
+  property http_sec_fetch_site : String?
+  property http_sec_fetch_user : String?
+  property http_sec_purpose : String?
 end

src/redis_server.cr

@@ -7,6 +7,7 @@ class RedisServer
   NOT_IMPLEMENTED = "-Not implemented#{CRLF}"
   QUIT_COMMAND = "quit"
   GET_COMMAND = "get"
+  NULL = "_#{CRLF}"
 
   property server : TCPServer
 
@@ -56,4 +57,8 @@ class RedisServer
   def self.bulk_string(client : TCPSocket, string : String)
     client.print(BULK_STRING, string.size, CRLF, string, CRLF)
   end
+
+  def self.null(client : TCPSocket)
+    client.print(NULL)
+  end
 end