v0.7.0

README.md

@@ -86,58 +86,8 @@ make alpine-build
 
 ## Database configuration
 
-Create an `access_logs` database with the following schema:
-
-| Field                               | Type    | Reference                            | Index? |
-| -----                               | ----    | ---------                            | ------ |
-| id                                  | String  | UUID                                 | Unique |
-| host                                | String  | Host name                            | Yes    |
-| msec                                | Float   | Unix timestamp of visit              | ?      |
-| server_protocol                     | String  | HTTP/Version                         | ?      |
-| request_method                      | String  | GET/POST/etc.                        | ?      |
-| request_completion                  | String  | "OK"                                 | ?      |
-| uri                                 | String  | Request                              | True   |
-| query_string                        | String  | Arguments                            | ?      |
-| status                              | Integer | HTTP status                          | ?      |
-| sent_http_content_type              | String  | MIME type of response                | ?      |
-| sent_http_content_encoding          | String  | Compression                          | ?      |
-| sent_http_etag                      | String  | ETag header                          | ?      |
-| sent_http_last_modified             | String  | Last modified date                   | ?      |
-| http_accept                         | String  | MIME types requested                 | ?      |
-| http_accept_encoding                | String  | Compression accepted                 | ?      |
-| http_accept_language                | String  | Languages supported                  | ?      |
-| http_pragma                         | String  | Pragma header                        | ?      |
-| http_cache_control                  | String  | Cache requested                      | ?      |
-| http_if_none_match                  | String  | ETag requested                       | ?      |
-| http_dnt                            | String  | Do Not Track header                  | ?      |
-| http_user_agent                     | String  | User Agent                           | Yes    |
-| http_origin                         | String  | Request origin                       | Yes    |
-| http_referer                        | String  | Referer (see Referrer Policy)        | Yes    |
-| request_time                        | Float   | Request duration                     | ?      |
-| bytes_sent                          | Integer | Bytes sent                           | ?      |
-| body_bytes_sent                     | Integer | Bytes sent not including headers     | ?      |
-| request_length                      | Integer | Headers                              | ?      |
-| http_connection                     | String  | Connection status                    | ?      |
-| pipe                                | String  | Connection was multiplexed           | ?      |
-| connection_requests                 | Integer | Requests done on the same connection | ?      |
-| geoip2_data_country_name            | String  | Country according to GeoIP           | Yes    |
-| geoip2_data_city_name               | String  | City according to GeoIP              | Yes    |
-| ssl_server_name                     | String  | SNI                                  | ?      |
-| ssl_protocol                        | String  | SSL/TLS version used                 | ?      |
-| ssl_early_data                      | String  | TLSv1.3 early data used              | ?      |
-| ssl_session_reused                  | String  | TLS session reused                   | ?      |
-| ssl_curves                          | String  | Curves used                          | ?      |
-| ssl_ciphers                         | String  | Ciphers available                    | ?      |
-| ssl_cipher                          | String  | Cipher used                          | ?      |
-| sent_http_x_xss_protection          | String  | XSS Protection sent                  | ?      |
-| sent_http_x_frame_options           | String  | Frame protection sent                | ?      |
-| sent_http_x_content_type_options    | String  | Content protection sent              | ?      |
-| sent_http_strict_transport_security | String  | HSTS sent                            | ?      |
-| nginx_version                       | String  | Server version                       | ?      |
-| pid                                 | Integer | Server PID                           | ?      |
-| crawler                             | Boolean | Web crawler detected                 | ?      |
-| remote_user                         | String  | HTTP Basic auth user                 | ?      |
-
+Create an `access_logs` database with an schema similar to
+[create.sql](contrib/schema.sql)
 
 ## Nginx configuration
 

contrib/asn.sql

@@ -0,0 +1,12 @@
+BEGIN;
+CREATE TABLE IF NOT EXISTS asn_import (range_start INTEGER, range_end INTEGER, id INTEGER, country STRING, description TEXT);
+CREATE TABLE IF NOT EXISTS asns (range_start INTEGER, range_end INTEGER, id INTEGER, country STRING, owner_id INTEGER);
+CREATE UNIQUE INDEX IF NOT EXISTS asn_import_range_id ON asn_import (range_start, range_end);
+CREATE UNIQUE INDEX IF NOT EXISTS asns_range_id ON asns (range_start, range_end);
+CREATE INDEX IF NOT EXISTS asns_id_idx ON asns (id);
+CREATE INDEX IF NOT EXISTS asns_owner_id_idx ON asns (owner_id);
+CREATE TABLE IF NOT EXISTS owners (id INTEGER PRIMARY KEY, description TEXT);
+CREATE TABLE IF NOT EXISTS status (id INTEGER PRIMARY KEY, blocked BOOLEAN DEFAULT FALSE);
+CREATE TABLE IF NOT EXISTS owner_status (id INTEGER PRIMARY KEY, blocked BOOLEAN DEFAULT FALSE);
+DELETE FROM asn_import;
+COMMIT;

contrib/asn_2.sql

@@ -0,0 +1,12 @@
+BEGIN;
+INSERT INTO owners (description) SELECT DISTINCT asn_import.description FROM asn_import LEFT JOIN owners ON asn_import.description = owners.description WHERE owners.description IS NULL;
+INSERT INTO asns (range_start, range_end, id, country, owner_id)
+  SELECT asn_import.range_start, asn_import.range_end, asn_import.id, asn_import.country, owners.id
+  FROM asn_import
+  LEFT JOIN owners
+  ON owners.description = asn_import.description
+ON CONFLICT DO UPDATE SET id = excluded.id, country = excluded.country, owner_id = excluded.owner_id;
+DROP TABLE asn_import;
+DROP INDEX asns_range_id;
+COMMIT;
+VACUUM;

contrib/asn_database.sh

@@ -1,11 +1,10 @@
 #!/bin/sh
 set -e
 
+cr="$(dirname "$(readlink -f "$0")")"
 db="${1:-$PWD/asn.sqlite3}"
-sqlite3 "$db" "CREATE TABLE IF NOT EXISTS asn (range_start INTEGER, range_end INTEGER, id INTEGER, country STRING, description TEXT);"
-sqlite3 "$db" "CREATE INDEX IF NOT EXISTS asn_id_idx ON asn (id);"
-sqlite3 "$db" "CREATE TABLE IF NOT EXISTS status (id INTEGER PRIMARY KEY, blocked BOOLEAN DEFAULT FALSE);"
-sqlite3 "$db" "DELETE from asn;"
-wget https://iptoasn.com/data/ip2asn-v4-u32.tsv.gz -O - | gunzip | sqlite3 -tabs "$db" ".import '|cat -' asn"
+cat "$cr/asn.sql" | sqlite3 "$db"
+wget https://iptoasn.com/data/ip2asn-v4-u32.tsv.gz -O - | gunzip | sqlite3 -tabs "$db" ".import '|cat -' asn_import"
+cat "$cr/asn_2.sql" | sqlite3 "$db"
 
 echo "ASN database created/updated at $db"

contrib/create.sql

@@ -56,7 +56,14 @@ CREATE TABLE IF NOT EXISTS "access_logs" (
  "total_co2" float DEFAULT NULL,
  "node" varchar DEFAULT NULL,
  "asn" integer DEFAULT NULL,
- "http3" varchar(2) DEFAULT NULL
+ "http3" varchar(2) DEFAULT NULL,
+ "http_sec_fetch_mode" varchar DEFAULT NULL,
+ "http_sec_fetch_dest" varchar DEFAULT NULL,
+ "http_sec_fetch_site" varchar DEFAULT NULL,
+ "http_sec_fetch_user" varchar DEFAULT NULL,
+ "http_sec_purpose" varchar DEFAULT NULL,
+ "limit_req_status" varchar DEFAULT NULL,
+ "limit_conn_status" varchar DEFAULT NULL
 );
 
 CREATE INDEX IF NOT EXISTS "index_access_logs_on_host" ON "access_logs" ("host");

shard.lock

@@ -1,5 +1,9 @@
 version: 2.0
 shards:
+  blank:
+    git: https://github.com/kostya/blank.git
+    version: 0.2.0
+
   db:
     git: https://github.com/crystal-lang/crystal-db.git
     version: 0.11.0

shard.yml

@@ -1,5 +1,5 @@
 name: "access_log"
-version: "0.6.1"
+version: "0.7.0"
 authors:
 - "f <f@sutty.nl>"
 targets:
@@ -12,3 +12,5 @@ dependencies:
     github: "will/crystal-pg"
   sqlite3:
     github: "crystal-lang/crystal-sqlite3"
+  blank:
+    github: "kostya/blank"

src/access_log.cr

@@ -9,10 +9,12 @@ require "uuid"
 require "system"
 require "./models/access_log"
 require "./models/crawler"
+require "./models/ai_bot"
 require "./swd"
 require "./asn"
+require "./presence"
 
-VERSION = "0.6.1"
+VERSION = "0.7.0"
 
 Log.setup_from_env
 
@@ -24,11 +26,13 @@ database = "sqlite3://./development.sqlite3"
 asn_database = ""
 # Detect web crawlers
 crawler = false
+ai_bot = false
 # Parse the crawlers repository
 crawlers = [] of Crawler
+ai_bots = IaBots.new
 # Fields in database
 # TODO: Obtain them from AccessLog
-fields = %w[id remote_user host msec server_protocol request_method request_completion uri request_uri query_string status sent_http_content_type sent_http_content_encoding sent_http_etag sent_http_last_modified http_accept http_accept_encoding http_accept_language http_pragma http_cache_control http_if_none_match http_dnt http_user_agent http_origin http_referer request_time bytes_sent body_bytes_sent request_length http_connection pipe connection_requests geoip2_data_country_name geoip2_data_city_name ssl_server_name ssl_protocol ssl_early_data ssl_session_reused ssl_curves ssl_ciphers ssl_cipher sent_http_x_xss_protection sent_http_x_frame_options sent_http_x_content_type_options sent_http_strict_transport_security nginx_version pid crawler datacenter_co2 network_co2 consumer_device_co2 production_co2 total_co2 node asn http3]
+fields = %w[id remote_user host msec server_protocol request_method request_completion uri request_uri query_string status sent_http_content_type sent_http_content_encoding sent_http_etag sent_http_last_modified http_accept http_accept_encoding http_accept_language http_pragma http_cache_control http_if_none_match http_dnt http_user_agent http_origin http_referer request_time bytes_sent body_bytes_sent request_length http_connection pipe connection_requests geoip2_data_country_name geoip2_data_city_name ssl_server_name ssl_protocol ssl_early_data ssl_session_reused ssl_curves ssl_ciphers ssl_cipher sent_http_x_xss_protection sent_http_x_frame_options sent_http_x_content_type_options sent_http_strict_transport_security nginx_version pid crawler datacenter_co2 network_co2 consumer_device_co2 production_co2 total_co2 node asn http3 http_sec_fetch_mode http_sec_fetch_dest http_sec_fetch_site http_sec_fetch_user http_sec_purpose limit_req_status limit_conn_status ai_bot]
 # Params for the query
 params = [] of String
 # SWD
@@ -68,6 +72,12 @@ OptionParser.parse do |p|
     crawlers = Array(Crawler).from_json File.read(c)
   end
 
+  p.on "-i ai-robots.json", "--ia-bots ai-robots.json", "AI bots repository" do |c|
+    crawler = true
+    ai_bot = true
+    ai_bots = IaBots.from_json File.read(c)
+  end
+
   p.on "-s /tmp/access_log.socket", "--socket /tmp/access_log.socket", "Listening socket" do |s|
     socket = s
   end
@@ -117,6 +127,11 @@ if crawler
   crawler_re = Regex.union(crawlers.map { |c| c.pattern })
 end
 
+if ai_bots
+  ai_bot_re = Regex.union(ai_bots.keys)
+  crawler_re = Regex.union(crawler_re, ai_bot_re)
+end
+
 def remove_socket!(socket)
   FileUtils.rm(socket) if File.exists? socket
 end
@@ -167,52 +182,52 @@ DB.open database do |db|
       # AccessLog as a NamedTuple.
       db.exec query,
               UUID.random.to_s,
-              access_log.remote_user,
-              access_log.host,
-              access_log.msec,
-              access_log.server_protocol,
-              access_log.request_method,
-              access_log.request_completion,
-              access_log.uri,
-              access_log.request_uri,
-              access_log.query_string,
+              access_log.remote_user.presence,
+              access_log.host.presence,
+              access_log.msec.presence,
+              access_log.server_protocol.presence,
+              access_log.request_method.presence,
+              access_log.request_completion.presence,
+              access_log.uri.presence,
+              access_log.request_uri.presence,
+              access_log.query_string.presence,
               access_log.status,
-              access_log.sent_http_content_type,
-              access_log.sent_http_content_encoding,
-              access_log.sent_http_etag,
-              access_log.sent_http_last_modified,
-              access_log.http_accept,
-              access_log.http_accept_encoding,
-              access_log.http_accept_language,
-              access_log.http_pragma,
-              access_log.http_cache_control,
-              access_log.http_if_none_match,
-              access_log.http_dnt,
-              access_log.http_user_agent,
-              access_log.http_origin,
-              access_log.http_referer,
+              access_log.sent_http_content_type.presence,
+              access_log.sent_http_content_encoding.presence,
+              access_log.sent_http_etag.presence,
+              access_log.sent_http_last_modified.presence,
+              access_log.http_accept.presence,
+              access_log.http_accept_encoding.presence,
+              access_log.http_accept_language.presence,
+              access_log.http_pragma.presence,
+              access_log.http_cache_control.presence,
+              access_log.http_if_none_match.presence,
+              access_log.http_dnt.presence,
+              access_log.http_user_agent.presence,
+              access_log.http_origin.presence,
+              access_log.http_referer.presence,
               access_log.request_time,
               access_log.bytes_sent,
               access_log.body_bytes_sent,
               access_log.request_length,
-              access_log.http_connection,
-              access_log.pipe,
+              access_log.http_connection.presence,
+              access_log.pipe.presence,
               access_log.connection_requests,
-              access_log.geoip2_data_country_name,
-              access_log.geoip2_data_city_name,
-              access_log.ssl_server_name,
-              access_log.ssl_protocol,
-              access_log.ssl_early_data,
-              access_log.ssl_session_reused,
-              access_log.ssl_curves,
-              access_log.ssl_ciphers,
-              access_log.ssl_cipher,
-              access_log.sent_http_x_xss_protection,
-              access_log.sent_http_x_frame_options,
-              access_log.sent_http_x_content_type_options,
-              access_log.sent_http_strict_transport_security,
-              access_log.nginx_version,
-              access_log.pid,
+              access_log.geoip2_data_country_name.presence,
+              access_log.geoip2_data_city_name.presence,
+              access_log.ssl_server_name.presence,
+              access_log.ssl_protocol.presence,
+              access_log.ssl_early_data.presence,
+              access_log.ssl_session_reused.presence,
+              access_log.ssl_curves.presence,
+              access_log.ssl_ciphers.presence,
+              access_log.ssl_cipher.presence,
+              access_log.sent_http_x_xss_protection.presence,
+              access_log.sent_http_x_frame_options.presence,
+              access_log.sent_http_x_content_type_options.presence,
+              access_log.sent_http_strict_transport_security.presence,
+              access_log.nginx_version.presence,
+              access_log.pid.presence,
               (crawler ? !!(crawler_re =~ access_log.http_user_agent) : false),
               (swd ? s.try(&.datacenter_co2) : nil),
               (swd ? s.try(&.network_co2) : nil),
@@ -221,7 +236,15 @@ DB.open database do |db|
               (swd ? s.try(&.total_co2) : nil),
               (node ? System.hostname : nil),
               a.try(&.id),
-              access_log.http3
+              access_log.http3.presence,
+              access_log.http_sec_fetch_mode.presence,
+              access_log.http_sec_fetch_dest.presence,
+              access_log.http_sec_fetch_site.presence,
+              access_log.http_sec_fetch_user.presence,
+              access_log.http_sec_purpose.presence,
+              access_log.limit_req_status.presence,
+              access_log.limit_conn_status.presence,
+              (ai_bot ? !!(ai_bot_re =~ access_log.http_user_agent) : false)
 
     # Ignore parsing errors
     rescue e : JSON::ParseException

src/asn.cr

@@ -6,13 +6,16 @@ class AsnResult
 
   property id : Int64?
   property country : String?
+  # TODO: Deprecate?
   property description : String?
+  property owner_id : Int64?
   property blocked : Bool?
 end
 
 class ASN
-  QUERY = "select id, country, description from asn where ? between range_start and range_end limit 1;"
+  QUERY = "select asns.id, country, owners.description, owner_id from asns left join owners on owners.id = asns.owner_id where ? between range_start and range_end limit 1;"
   STATUS = "select blocked from status where id = ? limit 1;"
+  OWNER_STATUS = "select blocked from owner_status where id = ? limit 1;"
 
   property db : DB::Database
 
@@ -33,7 +36,8 @@ class ASN
 
   def query(binary_address : UInt32) : AsnResult?
     (@cache[binary_address] ||= @db.query_one(QUERY, binary_address.to_i64, &.read(AsnResult))).tap do |result|
-      result.blocked = @db.query_one(STATUS, result.id, &.read(Bool))
+      result.blocked   = rescue_query { @db.query_one(STATUS, result.id, &.read(Bool)) }
+      result.blocked ||= rescue_query { @db.query_one(OWNER_STATUS, result.owner_id, &.read(Bool)) }
     end
   rescue DB::NoResultsError
     nil
@@ -42,4 +46,10 @@ class ASN
   def close
     @db.close
   end
+
+  def rescue_query(&)
+    yield
+  rescue DB::NoResultsError
+    nil
+  end
 end

src/ip2asn_server.cr

@@ -16,6 +16,7 @@ Log.setup_from_env
 VERSION = "0.2.0"
 ASN_ID = "id"
 ASN_DESCRIPTION = "description"
+ASN_OWNER_ID = "owner_id"
 ASN_COUNTRY = "country"
 ASN_BLOCKED = "blocked"
 
@@ -70,6 +71,7 @@ RedisServer.new(interface, port) do |client, _, key|
   response =
     case field
     when ASN_ID then result.try(&.id).try(&.to_s)
+    when ASN_OWNER_ID then result.try(&.owner_id).try(&.to_s)
     when ASN_COUNTRY then result.try(&.country)
     when ASN_DESCRIPTION then result.try(&.description)
     when ASN_BLOCKED then
@@ -81,10 +83,11 @@ RedisServer.new(interface, port) do |client, _, key|
     else nil
     end
 
-  next if response.nil?
-
-  # Reply
-  RedisServer.bulk_string(client, response)
+  if response.nil?
+    RedisServer.null(client)
+  else
+    RedisServer.bulk_string(client, response)
+  end
 end
 
 asn.close

src/models/access_log.cr

@@ -60,4 +60,11 @@ class AccessLog
   property pid : String
   property remote_addr : String?
   property http3 : String?
+  property http_sec_fetch_mode : String?
+  property http_sec_fetch_dest : String?
+  property http_sec_fetch_site : String?
+  property http_sec_fetch_user : String?
+  property http_sec_purpose : String?
+  property limit_req_status : String?
+  property limit_conn_status : String?
 end

src/models/ai_bot.cr

@@ -0,0 +1,13 @@
+require "json"
+
+class AiBot
+  include JSON::Serializable
+
+  property operator : String
+  property respect : String
+  property function : String
+  property frequency : String
+  property description : String
+end
+
+AiBots = Hash(String, AiBot)

src/presence.cr

@@ -0,0 +1,9 @@
+require "blank"
+
+class Object
+  def presence
+    return nil if blank?
+
+    self
+  end
+end

src/redis_server.cr

@@ -7,6 +7,7 @@ class RedisServer
   NOT_IMPLEMENTED = "-Not implemented#{CRLF}"
   QUIT_COMMAND = "quit"
   GET_COMMAND = "get"
+  NULL = "_#{CRLF}"
 
   property server : TCPServer
 
@@ -56,4 +57,8 @@ class RedisServer
   def self.bulk_string(client : TCPSocket, string : String)
     client.print(BULK_STRING, string.size, CRLF, string, CRLF)
   end
+
+  def self.null(client : TCPSocket)
+    client.print(NULL)
+  end
 end