Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 fauno
|
73cd424f70
|
||
|
fauno
|
f3df1d7fdb
|
||
|
fauno
|
2bf97c1f71
|
||
|
fauno
|
74928b740a
|
||
|
fauno
|
9c5257b590
|
@@ -86,58 +86,8 @@ make alpine-build
|
||||
|
||||
## Database configuration
|
||||
|
||||
Create an `access_logs` database with the following schema:
|
||||
|
||||
| Field | Type | Reference | Index? |
|
||||
| ----- | ---- | --------- | ------ |
|
||||
| id | String | UUID | Unique |
|
||||
| host | String | Host name | Yes |
|
||||
| msec | Float | Unix timestamp of visit | ? |
|
||||
| server_protocol | String | HTTP/Version | ? |
|
||||
| request_method | String | GET/POST/etc. | ? |
|
||||
| request_completion | String | "OK" | ? |
|
||||
| uri | String | Request | True |
|
||||
| query_string | String | Arguments | ? |
|
||||
| status | Integer | HTTP status | ? |
|
||||
| sent_http_content_type | String | MIME type of response | ? |
|
||||
| sent_http_content_encoding | String | Compression | ? |
|
||||
| sent_http_etag | String | ETag header | ? |
|
||||
| sent_http_last_modified | String | Last modified date | ? |
|
||||
| http_accept | String | MIME types requested | ? |
|
||||
| http_accept_encoding | String | Compression accepted | ? |
|
||||
| http_accept_language | String | Languages supported | ? |
|
||||
| http_pragma | String | Pragma header | ? |
|
||||
| http_cache_control | String | Cache requested | ? |
|
||||
| http_if_none_match | String | ETag requested | ? |
|
||||
| http_dnt | String | Do Not Track header | ? |
|
||||
| http_user_agent | String | User Agent | Yes |
|
||||
| http_origin | String | Request origin | Yes |
|
||||
| http_referer | String | Referer (see Referrer Policy) | Yes |
|
||||
| request_time | Float | Request duration | ? |
|
||||
| bytes_sent | Integer | Bytes sent | ? |
|
||||
| body_bytes_sent | Integer | Bytes sent not including headers | ? |
|
||||
| request_length | Integer | Headers | ? |
|
||||
| http_connection | String | Connection status | ? |
|
||||
| pipe | String | Connection was multiplexed | ? |
|
||||
| connection_requests | Integer | Requests done on the same connection | ? |
|
||||
| geoip2_data_country_name | String | Country according to GeoIP | Yes |
|
||||
| geoip2_data_city_name | String | City according to GeoIP | Yes |
|
||||
| ssl_server_name | String | SNI | ? |
|
||||
| ssl_protocol | String | SSL/TLS version used | ? |
|
||||
| ssl_early_data | String | TLSv1.3 early data used | ? |
|
||||
| ssl_session_reused | String | TLS session reused | ? |
|
||||
| ssl_curves | String | Curves used | ? |
|
||||
| ssl_ciphers | String | Ciphers available | ? |
|
||||
| ssl_cipher | String | Cipher used | ? |
|
||||
| sent_http_x_xss_protection | String | XSS Protection sent | ? |
|
||||
| sent_http_x_frame_options | String | Frame protection sent | ? |
|
||||
| sent_http_x_content_type_options | String | Content protection sent | ? |
|
||||
| sent_http_strict_transport_security | String | HSTS sent | ? |
|
||||
| nginx_version | String | Server version | ? |
|
||||
| pid | Integer | Server PID | ? |
|
||||
| crawler | Boolean | Web crawler detected | ? |
|
||||
| remote_user | String | HTTP Basic auth user | ? |
|
||||
|
||||
Create an `access_logs` database with an schema similar to
|
||||
[create.sql](contrib/schema.sql)
|
||||
|
||||
## Nginx configuration
|
||||
|
||||
|
||||
+3
-1
@@ -61,7 +61,9 @@ CREATE TABLE IF NOT EXISTS "access_logs" (
|
||||
"http_sec_fetch_dest" varchar DEFAULT NULL,
|
||||
"http_sec_fetch_site" varchar DEFAULT NULL,
|
||||
"http_sec_fetch_user" varchar DEFAULT NULL,
|
||||
"http_sec_purpose" varchar DEFAULT NULL
|
||||
"http_sec_purpose" varchar DEFAULT NULL,
|
||||
"limit_req_status" varchar DEFAULT NULL,
|
||||
"limit_conn_status" varchar DEFAULT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "index_access_logs_on_host" ON "access_logs" ("host");
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
name: "access_log"
|
||||
version: "0.6.2"
|
||||
version: "0.7.0"
|
||||
authors:
|
||||
- "f <f@sutty.nl>"
|
||||
targets:
|
||||
|
||||
+20
-3
@@ -9,11 +9,12 @@ require "uuid"
|
||||
require "system"
|
||||
require "./models/access_log"
|
||||
require "./models/crawler"
|
||||
require "./models/ai_bot"
|
||||
require "./swd"
|
||||
require "./asn"
|
||||
require "./presence"
|
||||
|
||||
VERSION = "0.6.4"
|
||||
VERSION = "0.7.0"
|
||||
|
||||
Log.setup_from_env
|
||||
|
||||
@@ -25,11 +26,13 @@ database = "sqlite3://./development.sqlite3"
|
||||
asn_database = ""
|
||||
# Detect web crawlers
|
||||
crawler = false
|
||||
ai_bot = false
|
||||
# Parse the crawlers repository
|
||||
crawlers = [] of Crawler
|
||||
ai_bots = IaBots.new
|
||||
# Fields in database
|
||||
# TODO: Obtain them from AccessLog
|
||||
fields = %w[id remote_user host msec server_protocol request_method request_completion uri request_uri query_string status sent_http_content_type sent_http_content_encoding sent_http_etag sent_http_last_modified http_accept http_accept_encoding http_accept_language http_pragma http_cache_control http_if_none_match http_dnt http_user_agent http_origin http_referer request_time bytes_sent body_bytes_sent request_length http_connection pipe connection_requests geoip2_data_country_name geoip2_data_city_name ssl_server_name ssl_protocol ssl_early_data ssl_session_reused ssl_curves ssl_ciphers ssl_cipher sent_http_x_xss_protection sent_http_x_frame_options sent_http_x_content_type_options sent_http_strict_transport_security nginx_version pid crawler datacenter_co2 network_co2 consumer_device_co2 production_co2 total_co2 node asn http3 http_sec_fetch_mode http_sec_fetch_dest http_sec_fetch_site http_sec_fetch_user http_sec_purpose]
|
||||
fields = %w[id remote_user host msec server_protocol request_method request_completion uri request_uri query_string status sent_http_content_type sent_http_content_encoding sent_http_etag sent_http_last_modified http_accept http_accept_encoding http_accept_language http_pragma http_cache_control http_if_none_match http_dnt http_user_agent http_origin http_referer request_time bytes_sent body_bytes_sent request_length http_connection pipe connection_requests geoip2_data_country_name geoip2_data_city_name ssl_server_name ssl_protocol ssl_early_data ssl_session_reused ssl_curves ssl_ciphers ssl_cipher sent_http_x_xss_protection sent_http_x_frame_options sent_http_x_content_type_options sent_http_strict_transport_security nginx_version pid crawler datacenter_co2 network_co2 consumer_device_co2 production_co2 total_co2 node asn http3 http_sec_fetch_mode http_sec_fetch_dest http_sec_fetch_site http_sec_fetch_user http_sec_purpose limit_req_status limit_conn_status ai_bot]
|
||||
# Params for the query
|
||||
params = [] of String
|
||||
# SWD
|
||||
@@ -69,6 +72,12 @@ OptionParser.parse do |p|
|
||||
crawlers = Array(Crawler).from_json File.read(c)
|
||||
end
|
||||
|
||||
p.on "-i ai-robots.json", "--ia-bots ai-robots.json", "AI bots repository" do |c|
|
||||
crawler = true
|
||||
ai_bot = true
|
||||
ai_bots = IaBots.from_json File.read(c)
|
||||
end
|
||||
|
||||
p.on "-s /tmp/access_log.socket", "--socket /tmp/access_log.socket", "Listening socket" do |s|
|
||||
socket = s
|
||||
end
|
||||
@@ -118,6 +127,11 @@ if crawler
|
||||
crawler_re = Regex.union(crawlers.map { |c| c.pattern })
|
||||
end
|
||||
|
||||
if ai_bots
|
||||
ai_bot_re = Regex.union(ai_bots.keys)
|
||||
crawler_re = Regex.union(crawler_re, ai_bot_re)
|
||||
end
|
||||
|
||||
def remove_socket!(socket)
|
||||
FileUtils.rm(socket) if File.exists? socket
|
||||
end
|
||||
@@ -227,7 +241,10 @@ DB.open database do |db|
|
||||
access_log.http_sec_fetch_dest.presence,
|
||||
access_log.http_sec_fetch_site.presence,
|
||||
access_log.http_sec_fetch_user.presence,
|
||||
access_log.http_sec_purpose.presence
|
||||
access_log.http_sec_purpose.presence,
|
||||
access_log.limit_req_status.presence,
|
||||
access_log.limit_conn_status.presence,
|
||||
(ai_bot ? !!(ai_bot_re =~ access_log.http_user_agent) : false)
|
||||
|
||||
# Ignore parsing errors
|
||||
rescue e : JSON::ParseException
|
||||
|
||||
@@ -65,4 +65,6 @@ class AccessLog
|
||||
property http_sec_fetch_site : String?
|
||||
property http_sec_fetch_user : String?
|
||||
property http_sec_purpose : String?
|
||||
property limit_req_status : String?
|
||||
property limit_conn_status : String?
|
||||
end
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
require "json"
|
||||
|
||||
class AiBot
|
||||
include JSON::Serializable
|
||||
|
||||
property operator : String
|
||||
property respect : String
|
||||
property function : String
|
||||
property frequency : String
|
||||
property description : String
|
||||
end
|
||||
|
||||
AiBots = Hash(String, AiBot)
|
||||
Reference in New Issue
Block a user