Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 fauno
|
db0406cec1
|
||
|
fauno
|
56ddd6c250
|
||
|
fauno
|
73cd424f70
|
||
|
fauno
|
f3df1d7fdb
|
||
|
fauno
|
2bf97c1f71
|
||
|
fauno
|
74928b740a
|
||
|
fauno
|
9c5257b590
|
||
|
fauno
|
b4e6a0100a
|
||
|
fauno
|
b86f0318a3
|
||
|
fauno
|
9fdf239422
|
@@ -86,58 +86,8 @@ make alpine-build
|
||||
|
||||
## Database configuration
|
||||
|
||||
Create an `access_logs` database with the following schema:
|
||||
|
||||
| Field | Type | Reference | Index? |
|
||||
| ----- | ---- | --------- | ------ |
|
||||
| id | String | UUID | Unique |
|
||||
| host | String | Host name | Yes |
|
||||
| msec | Float | Unix timestamp of visit | ? |
|
||||
| server_protocol | String | HTTP/Version | ? |
|
||||
| request_method | String | GET/POST/etc. | ? |
|
||||
| request_completion | String | "OK" | ? |
|
||||
| uri | String | Request | True |
|
||||
| query_string | String | Arguments | ? |
|
||||
| status | Integer | HTTP status | ? |
|
||||
| sent_http_content_type | String | MIME type of response | ? |
|
||||
| sent_http_content_encoding | String | Compression | ? |
|
||||
| sent_http_etag | String | ETag header | ? |
|
||||
| sent_http_last_modified | String | Last modified date | ? |
|
||||
| http_accept | String | MIME types requested | ? |
|
||||
| http_accept_encoding | String | Compression accepted | ? |
|
||||
| http_accept_language | String | Languages supported | ? |
|
||||
| http_pragma | String | Pragma header | ? |
|
||||
| http_cache_control | String | Cache requested | ? |
|
||||
| http_if_none_match | String | ETag requested | ? |
|
||||
| http_dnt | String | Do Not Track header | ? |
|
||||
| http_user_agent | String | User Agent | Yes |
|
||||
| http_origin | String | Request origin | Yes |
|
||||
| http_referer | String | Referer (see Referrer Policy) | Yes |
|
||||
| request_time | Float | Request duration | ? |
|
||||
| bytes_sent | Integer | Bytes sent | ? |
|
||||
| body_bytes_sent | Integer | Bytes sent not including headers | ? |
|
||||
| request_length | Integer | Headers | ? |
|
||||
| http_connection | String | Connection status | ? |
|
||||
| pipe | String | Connection was multiplexed | ? |
|
||||
| connection_requests | Integer | Requests done on the same connection | ? |
|
||||
| geoip2_data_country_name | String | Country according to GeoIP | Yes |
|
||||
| geoip2_data_city_name | String | City according to GeoIP | Yes |
|
||||
| ssl_server_name | String | SNI | ? |
|
||||
| ssl_protocol | String | SSL/TLS version used | ? |
|
||||
| ssl_early_data | String | TLSv1.3 early data used | ? |
|
||||
| ssl_session_reused | String | TLS session reused | ? |
|
||||
| ssl_curves | String | Curves used | ? |
|
||||
| ssl_ciphers | String | Ciphers available | ? |
|
||||
| ssl_cipher | String | Cipher used | ? |
|
||||
| sent_http_x_xss_protection | String | XSS Protection sent | ? |
|
||||
| sent_http_x_frame_options | String | Frame protection sent | ? |
|
||||
| sent_http_x_content_type_options | String | Content protection sent | ? |
|
||||
| sent_http_strict_transport_security | String | HSTS sent | ? |
|
||||
| nginx_version | String | Server version | ? |
|
||||
| pid | Integer | Server PID | ? |
|
||||
| crawler | Boolean | Web crawler detected | ? |
|
||||
| remote_user | String | HTTP Basic auth user | ? |
|
||||
|
||||
Create an `access_logs` database with an schema similar to
|
||||
[create.sql](contrib/schema.sql)
|
||||
|
||||
## Nginx configuration
|
||||
|
||||
|
||||
+3
-1
@@ -61,7 +61,9 @@ CREATE TABLE IF NOT EXISTS "access_logs" (
|
||||
"http_sec_fetch_dest" varchar DEFAULT NULL,
|
||||
"http_sec_fetch_site" varchar DEFAULT NULL,
|
||||
"http_sec_fetch_user" varchar DEFAULT NULL,
|
||||
"http_sec_purpose" varchar DEFAULT NULL
|
||||
"http_sec_purpose" varchar DEFAULT NULL,
|
||||
"limit_req_status" varchar DEFAULT NULL,
|
||||
"limit_conn_status" varchar DEFAULT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "index_access_logs_on_host" ON "access_logs" ("host");
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
version: 2.0
|
||||
shards:
|
||||
blank:
|
||||
git: https://github.com/kostya/blank.git
|
||||
version: 0.2.0
|
||||
|
||||
db:
|
||||
git: https://github.com/crystal-lang/crystal-db.git
|
||||
version: 0.11.0
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
name: "access_log"
|
||||
version: "0.6.2"
|
||||
version: "0.7.1"
|
||||
authors:
|
||||
- "f <f@sutty.nl>"
|
||||
targets:
|
||||
@@ -12,3 +12,5 @@ dependencies:
|
||||
github: "will/crystal-pg"
|
||||
sqlite3:
|
||||
github: "crystal-lang/crystal-sqlite3"
|
||||
blank:
|
||||
github: "kostya/blank"
|
||||
|
||||
+66
-48
@@ -9,10 +9,12 @@ require "uuid"
|
||||
require "system"
|
||||
require "./models/access_log"
|
||||
require "./models/crawler"
|
||||
require "./models/ai_bot"
|
||||
require "./swd"
|
||||
require "./asn"
|
||||
require "./presence"
|
||||
|
||||
VERSION = "0.6.3"
|
||||
VERSION = "0.7.1"
|
||||
|
||||
Log.setup_from_env
|
||||
|
||||
@@ -24,11 +26,13 @@ database = "sqlite3://./development.sqlite3"
|
||||
asn_database = ""
|
||||
# Detect web crawlers
|
||||
crawler = false
|
||||
ai_bot = false
|
||||
# Parse the crawlers repository
|
||||
crawlers = [] of Crawler
|
||||
ai_bots = AiBots.new
|
||||
# Fields in database
|
||||
# TODO: Obtain them from AccessLog
|
||||
fields = %w[id remote_user host msec server_protocol request_method request_completion uri request_uri query_string status sent_http_content_type sent_http_content_encoding sent_http_etag sent_http_last_modified http_accept http_accept_encoding http_accept_language http_pragma http_cache_control http_if_none_match http_dnt http_user_agent http_origin http_referer request_time bytes_sent body_bytes_sent request_length http_connection pipe connection_requests geoip2_data_country_name geoip2_data_city_name ssl_server_name ssl_protocol ssl_early_data ssl_session_reused ssl_curves ssl_ciphers ssl_cipher sent_http_x_xss_protection sent_http_x_frame_options sent_http_x_content_type_options sent_http_strict_transport_security nginx_version pid crawler datacenter_co2 network_co2 consumer_device_co2 production_co2 total_co2 node asn http3 http_sec_fetch_mode http_sec_fetch_dest http_sec_fetch_site http_sec_fetch_user http_sec_purpose]
|
||||
fields = %w[id remote_user host msec server_protocol request_method request_completion uri request_uri query_string status sent_http_content_type sent_http_content_encoding sent_http_etag sent_http_last_modified http_accept http_accept_encoding http_accept_language http_pragma http_cache_control http_if_none_match http_dnt http_user_agent http_origin http_referer request_time bytes_sent body_bytes_sent request_length http_connection pipe connection_requests geoip2_data_country_name geoip2_data_city_name ssl_server_name ssl_protocol ssl_early_data ssl_session_reused ssl_curves ssl_ciphers ssl_cipher sent_http_x_xss_protection sent_http_x_frame_options sent_http_x_content_type_options sent_http_strict_transport_security nginx_version pid crawler datacenter_co2 network_co2 consumer_device_co2 production_co2 total_co2 node asn http3 http_sec_fetch_mode http_sec_fetch_dest http_sec_fetch_site http_sec_fetch_user http_sec_purpose limit_req_status limit_conn_status ai_bot]
|
||||
# Params for the query
|
||||
params = [] of String
|
||||
# SWD
|
||||
@@ -68,6 +72,12 @@ OptionParser.parse do |p|
|
||||
crawlers = Array(Crawler).from_json File.read(c)
|
||||
end
|
||||
|
||||
p.on "-i ai-robots.json", "--ai-bots ai-robots.json", "AI bots repository" do |c|
|
||||
crawler = true
|
||||
ai_bot = true
|
||||
ai_bots = AiBots.from_json File.read(c)
|
||||
end
|
||||
|
||||
p.on "-s /tmp/access_log.socket", "--socket /tmp/access_log.socket", "Listening socket" do |s|
|
||||
socket = s
|
||||
end
|
||||
@@ -117,6 +127,11 @@ if crawler
|
||||
crawler_re = Regex.union(crawlers.map { |c| c.pattern })
|
||||
end
|
||||
|
||||
if ai_bots
|
||||
ai_bot_re = Regex.union(ai_bots.keys)
|
||||
crawler_re = Regex.union(crawler_re, ai_bot_re)
|
||||
end
|
||||
|
||||
def remove_socket!(socket)
|
||||
FileUtils.rm(socket) if File.exists? socket
|
||||
end
|
||||
@@ -167,52 +182,52 @@ DB.open database do |db|
|
||||
# AccessLog as a NamedTuple.
|
||||
db.exec query,
|
||||
UUID.random.to_s,
|
||||
access_log.remote_user,
|
||||
access_log.host,
|
||||
access_log.msec,
|
||||
access_log.server_protocol,
|
||||
access_log.request_method,
|
||||
access_log.request_completion,
|
||||
access_log.uri,
|
||||
access_log.request_uri,
|
||||
access_log.query_string,
|
||||
access_log.remote_user.presence,
|
||||
access_log.host.presence,
|
||||
access_log.msec.presence,
|
||||
access_log.server_protocol.presence,
|
||||
access_log.request_method.presence,
|
||||
access_log.request_completion.presence,
|
||||
access_log.uri.presence,
|
||||
access_log.request_uri.presence,
|
||||
access_log.query_string.presence,
|
||||
access_log.status,
|
||||
access_log.sent_http_content_type,
|
||||
access_log.sent_http_content_encoding,
|
||||
access_log.sent_http_etag,
|
||||
access_log.sent_http_last_modified,
|
||||
access_log.http_accept,
|
||||
access_log.http_accept_encoding,
|
||||
access_log.http_accept_language,
|
||||
access_log.http_pragma,
|
||||
access_log.http_cache_control,
|
||||
access_log.http_if_none_match,
|
||||
access_log.http_dnt,
|
||||
access_log.http_user_agent,
|
||||
access_log.http_origin,
|
||||
access_log.http_referer,
|
||||
access_log.sent_http_content_type.presence,
|
||||
access_log.sent_http_content_encoding.presence,
|
||||
access_log.sent_http_etag.presence,
|
||||
access_log.sent_http_last_modified.presence,
|
||||
access_log.http_accept.presence,
|
||||
access_log.http_accept_encoding.presence,
|
||||
access_log.http_accept_language.presence,
|
||||
access_log.http_pragma.presence,
|
||||
access_log.http_cache_control.presence,
|
||||
access_log.http_if_none_match.presence,
|
||||
access_log.http_dnt.presence,
|
||||
access_log.http_user_agent.presence,
|
||||
access_log.http_origin.presence,
|
||||
access_log.http_referer.presence,
|
||||
access_log.request_time,
|
||||
access_log.bytes_sent,
|
||||
access_log.body_bytes_sent,
|
||||
access_log.request_length,
|
||||
access_log.http_connection,
|
||||
access_log.pipe,
|
||||
access_log.http_connection.presence,
|
||||
access_log.pipe.presence,
|
||||
access_log.connection_requests,
|
||||
access_log.geoip2_data_country_name,
|
||||
access_log.geoip2_data_city_name,
|
||||
access_log.ssl_server_name,
|
||||
access_log.ssl_protocol,
|
||||
access_log.ssl_early_data,
|
||||
access_log.ssl_session_reused,
|
||||
access_log.ssl_curves,
|
||||
access_log.ssl_ciphers,
|
||||
access_log.ssl_cipher,
|
||||
access_log.sent_http_x_xss_protection,
|
||||
access_log.sent_http_x_frame_options,
|
||||
access_log.sent_http_x_content_type_options,
|
||||
access_log.sent_http_strict_transport_security,
|
||||
access_log.nginx_version,
|
||||
access_log.pid,
|
||||
access_log.geoip2_data_country_name.presence,
|
||||
access_log.geoip2_data_city_name.presence,
|
||||
access_log.ssl_server_name.presence,
|
||||
access_log.ssl_protocol.presence,
|
||||
access_log.ssl_early_data.presence,
|
||||
access_log.ssl_session_reused.presence,
|
||||
access_log.ssl_curves.presence,
|
||||
access_log.ssl_ciphers.presence,
|
||||
access_log.ssl_cipher.presence,
|
||||
access_log.sent_http_x_xss_protection.presence,
|
||||
access_log.sent_http_x_frame_options.presence,
|
||||
access_log.sent_http_x_content_type_options.presence,
|
||||
access_log.sent_http_strict_transport_security.presence,
|
||||
access_log.nginx_version.presence,
|
||||
access_log.pid.presence,
|
||||
(crawler ? !!(crawler_re =~ access_log.http_user_agent) : false),
|
||||
(swd ? s.try(&.datacenter_co2) : nil),
|
||||
(swd ? s.try(&.network_co2) : nil),
|
||||
@@ -221,12 +236,15 @@ DB.open database do |db|
|
||||
(swd ? s.try(&.total_co2) : nil),
|
||||
(node ? System.hostname : nil),
|
||||
a.try(&.id),
|
||||
access_log.http3,
|
||||
access_log.http_sec_fetch_mode,
|
||||
access_log.http_sec_fetch_dest,
|
||||
access_log.http_sec_fetch_site,
|
||||
access_log.http_sec_fetch_user,
|
||||
access_log.http_sec_purpose
|
||||
access_log.http3.presence,
|
||||
access_log.http_sec_fetch_mode.presence,
|
||||
access_log.http_sec_fetch_dest.presence,
|
||||
access_log.http_sec_fetch_site.presence,
|
||||
access_log.http_sec_fetch_user.presence,
|
||||
access_log.http_sec_purpose.presence,
|
||||
access_log.limit_req_status.presence,
|
||||
access_log.limit_conn_status.presence,
|
||||
(ai_bot ? !!(ai_bot_re =~ access_log.http_user_agent) : false)
|
||||
|
||||
# Ignore parsing errors
|
||||
rescue e : JSON::ParseException
|
||||
|
||||
@@ -65,4 +65,6 @@ class AccessLog
|
||||
property http_sec_fetch_site : String?
|
||||
property http_sec_fetch_user : String?
|
||||
property http_sec_purpose : String?
|
||||
property limit_req_status : String?
|
||||
property limit_conn_status : String?
|
||||
end
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
require "json"
|
||||
|
||||
class AiBot
|
||||
include JSON::Serializable
|
||||
|
||||
property operator : String
|
||||
property respect : String
|
||||
property function : String
|
||||
property frequency : String
|
||||
property description : String
|
||||
end
|
||||
|
||||
AiBots = Hash(String, AiBot)
|
||||
@@ -0,0 +1,9 @@
|
||||
require "blank"
|
||||
|
||||
class Object
|
||||
def presence
|
||||
return nil if blank?
|
||||
|
||||
self
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user