v0.7.1

shard.yml

@@ -1,5 +1,5 @@
 name: "access_log"
-version: "0.6.5"
+version: "0.7.1"
 authors:
 - "f <f@sutty.nl>"
 targets:

src/access_log.cr

@@ -9,11 +9,12 @@ require "uuid"
 require "system"
 require "./models/access_log"
 require "./models/crawler"
+require "./models/ai_bot"
 require "./swd"
 require "./asn"
 require "./presence"
 
-VERSION = "0.6.5"
+VERSION = "0.7.1"
 
 Log.setup_from_env
 
@@ -25,11 +26,13 @@ database = "sqlite3://./development.sqlite3"
 asn_database = ""
 # Detect web crawlers
 crawler = false
+ai_bot = false
 # Parse the crawlers repository
 crawlers = [] of Crawler
+ai_bots = AiBots.new
 # Fields in database
 # TODO: Obtain them from AccessLog
-fields = %w[id remote_user host msec server_protocol request_method request_completion uri request_uri query_string status sent_http_content_type sent_http_content_encoding sent_http_etag sent_http_last_modified http_accept http_accept_encoding http_accept_language http_pragma http_cache_control http_if_none_match http_dnt http_user_agent http_origin http_referer request_time bytes_sent body_bytes_sent request_length http_connection pipe connection_requests geoip2_data_country_name geoip2_data_city_name ssl_server_name ssl_protocol ssl_early_data ssl_session_reused ssl_curves ssl_ciphers ssl_cipher sent_http_x_xss_protection sent_http_x_frame_options sent_http_x_content_type_options sent_http_strict_transport_security nginx_version pid crawler datacenter_co2 network_co2 consumer_device_co2 production_co2 total_co2 node asn http3 http_sec_fetch_mode http_sec_fetch_dest http_sec_fetch_site http_sec_fetch_user http_sec_purpose limit_req_status limit_conn_status]
+fields = %w[id remote_user host msec server_protocol request_method request_completion uri request_uri query_string status sent_http_content_type sent_http_content_encoding sent_http_etag sent_http_last_modified http_accept http_accept_encoding http_accept_language http_pragma http_cache_control http_if_none_match http_dnt http_user_agent http_origin http_referer request_time bytes_sent body_bytes_sent request_length http_connection pipe connection_requests geoip2_data_country_name geoip2_data_city_name ssl_server_name ssl_protocol ssl_early_data ssl_session_reused ssl_curves ssl_ciphers ssl_cipher sent_http_x_xss_protection sent_http_x_frame_options sent_http_x_content_type_options sent_http_strict_transport_security nginx_version pid crawler datacenter_co2 network_co2 consumer_device_co2 production_co2 total_co2 node asn http3 http_sec_fetch_mode http_sec_fetch_dest http_sec_fetch_site http_sec_fetch_user http_sec_purpose limit_req_status limit_conn_status ai_bot]
 # Params for the query
 params = [] of String
 # SWD
@@ -69,6 +72,12 @@ OptionParser.parse do |p|
     crawlers = Array(Crawler).from_json File.read(c)
   end
 
+  p.on "-i ai-robots.json", "--ai-bots ai-robots.json", "AI bots repository" do |c|
+    crawler = true
+    ai_bot = true
+    ai_bots = AiBots.from_json File.read(c)
+  end
+
   p.on "-s /tmp/access_log.socket", "--socket /tmp/access_log.socket", "Listening socket" do |s|
     socket = s
   end
@@ -118,6 +127,11 @@ if crawler
   crawler_re = Regex.union(crawlers.map { |c| c.pattern })
 end
 
+if ai_bots
+  ai_bot_re = Regex.union(ai_bots.keys)
+  crawler_re = Regex.union(crawler_re, ai_bot_re)
+end
+
 def remove_socket!(socket)
   FileUtils.rm(socket) if File.exists? socket
 end
@@ -229,7 +243,8 @@ DB.open database do |db|
               access_log.http_sec_fetch_user.presence,
               access_log.http_sec_purpose.presence,
               access_log.limit_req_status.presence,
-              access_log.limit_conn_status.presence
+              access_log.limit_conn_status.presence,
+              (ai_bot ? !!(ai_bot_re =~ access_log.http_user_agent) : false)
 
     # Ignore parsing errors
     rescue e : JSON::ParseException

src/models/ai_bot.cr

@@ -0,0 +1,13 @@
+require "json"
+
+class AiBot
+  include JSON::Serializable
+
+  property operator : String
+  property respect : String
+  property function : String
+  property frequency : String
+  property description : String
+end
+
+AiBots = Hash(String, AiBot)