ci: link_type

Resolve "Iniciar un socket unix"

Closes #2

See merge request sutty/access_log!2

.gitlab-ci.yml

@@ -0,0 +1,41 @@
+stages:
+- "build"
+- "upload"
+- "release"
+variables:
+  RELEASE_DIRECTORY: "access-log-${CI_COMMIT_TAG}-linux-amd64"
+  RELEASE_TARBALL: "${RELEASE_DIRECTORY}.tar.gz"
+  PACKAGE_REGISTRY_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/access_log/${CI_COMMIT_TAG}/${RELEASE_TARBALL}"
+build:
+  stage: "build"
+  image: "crystallang/crystal:latest-alpine"
+  rules:
+  - if: "$CI_COMMIT_TAG"
+  cache:
+    paths:
+    - "lib/"
+  script:
+  - "apk add sqlite-static"
+  - "shards install"
+  - "crystal build --release --static src/access_log.cr"
+  - "strip --strip-all access_log"
+  - "mkdir -p ${RELEASE_DIRECTORY}"
+  - "mv access_log LICENSE ${RELEASE_DIRECTORY}"
+  - "tar -czf ${RELEASE_TARBALL} ${RELEASE_DIRECTORY}"
+  artifacts:
+    paths:
+    - "${RELEASE_TARBALL}"
+upload:
+  stage: "upload"
+  image: "curlimages/curl:latest"
+  rules:
+  - if: "$CI_COMMIT_TAG"
+  script:
+  - "curl --header \"JOB-TOKEN: ${CI_JOB_TOKEN}\" --upload-file ${RELEASE_TARBALL} ${PACKAGE_REGISTRY_URL}"
+release:
+  stage: "release"
+  image: "registry.gitlab.com/gitlab-org/release-cli:latest"
+  rules:
+  - if: "$CI_COMMIT_TAG"
+  script:
+    - "release-cli create --name \"Release ${CI_COMMIT_TAG}\" --tag-name ${CI_COMMIT_TAG} --assets-link \"{\\\"name\\\":\\\"${RELEASE_TARBALL}\\\",\\\"url\\\":\\\"${PACKAGE_REGISTRY_URL}\\\",\\\"link_type\\\":\\\"package\\\"}\""

README.md

@@ -1,6 +1,6 @@
 # access_log
 
-Receives access logs from stdin in JSON format and stores them on
+Receives access logs on a UNIX socket in JSON format and stores them on
 a database.  It **intentionally** doesn't collect IP addresses.  It
 doesn't respect the Do Not Track (DNT) header though, because we're not
 collecting personally identifiable data.  Referrer collection is
@@ -111,8 +111,10 @@ Configure Nginx to format access log as JSON.  You can configure
 `http_referer` (double and single "r" respectively, the second is a typo
 on the HTTP specification).
 
-Install `daemonize` and run `access_logd` to create `access.log` as
-a FIFO node, so Nginx writes to it and `access_log` can read from it.
+Install `daemonize` and run `access_logd`.  By default it creates a UNIX
+socket on `/tmp/access_log.socket` so Nginx writes can write to it using
+its [syslog support](https://nginx.org/en/docs/syslog.html).
+
 Check `/var/log/nginx/error.log` for debugging.
 
 `ACCESS_LOG_FLAGS` is the env variable to pass flags to `access_logd`.
@@ -122,7 +124,7 @@ container](https://0xacab.org/sutty/containers/nginx/).
 ```json
 log_format main escape=json '{"host":"$host","msec":$msec,"server_protocol":"$server_protocol","request_method":"$request_method","request_completion":"$request_completion","uri":"$uri","query_string":"$query_string","status":$status,"sent_http_content_type":"$sent_http_content_type","sent_http_content_encoding":"$sent_http_content_encoding","sent_http_etag":"$sent_http_etag","sent_http_last_modified":"$sent_http_last_modified","http_accept":"$http_accept","http_accept_encoding":"$http_accept_encoding","http_accept_language":"$http_accept_language","http_pragma":"$http_pragma","http_cache_control":"$http_cache_control","http_if_none_match":"$http_if_none_match","http_dnt":"$http_dnt","http_user_agent":"$http_user_agent","http_origin":"$http_origin","http_referer":{"origin":"$http_origin","referrer":"$http_referer","policy":"origin-when-cross-origin"},"request_time":$request_time,"bytes_sent":$bytes_sent,"body_bytes_sent":$body_bytes_sent,"request_length":$request_length,"http_connection":"$http_connection","pipe":"$pipe","connection_requests":$connection_requests,"geoip2_data_country_name":"$geoip2_data_country_name","geoip2_data_city_name":"$geoip2_data_city_name","ssl_server_name":"$ssl_server_name","ssl_protocol":"$ssl_protocol","ssl_early_data":"$ssl_early_data","ssl_session_reused":"$ssl_session_reused","ssl_curves":"$ssl_curves","ssl_ciphers":"$ssl_ciphers","ssl_cipher":"$ssl_cipher","sent_http_x_xss_protection":"$sent_http_x_xss_protection","sent_http_x_frame_options":"$sent_http_x_frame_options","sent_http_x_content_type_options":"$sent_http_x_content_type_options","sent_http_strict_transport_security":"$sent_http_strict_transport_security","nginx_version":"$nginx_version","pid":"$pid","remote_user":""}';
 
-access_log /var/log/nginx/access.log main;
+access_log syslog=unix:/tmp/access_log.socket,nohostname main;
 ```
 
 ## Crawler user agents

access_logd

@@ -1,16 +1,11 @@
 #!/bin/sh
 
-# Fail if there's no options provided
+# Fail if there are no options provided
 test -n "$ACCESS_LOGS_FLAGS" || exit 1
 
-# Remove access.log and recreate it as a fifo
-rm -f /var/log/nginx/access.log /run/access_log.pid
-mkfifo /var/log/nginx/access.log
-chmod 755 /var/log/nginx/access.log
-
-# Reopen log file
-nginx -s reload
+# Give write access to Nginx
+umask 007
 
 # Read from fifo and load into database
-daemonize -p /run/access_logd.pid -u nobody \
-  /bin/sh -c "cat /var/log/nginx/access.log | access_log $ACCESS_LOGS_FLAGS"
+daemonize -p /run/access_logd.pid -u nobody -g www-data \
+  /usr/bin/access_log $ACCESS_LOGS_FLAGS

shard.lock

@@ -2,13 +2,13 @@ version: 2.0
 shards:
   db:
     git: https://github.com/crystal-lang/crystal-db.git
-    version: 0.9.0
+    version: 0.11.0
 
   pg:
     git: https://github.com/will/crystal-pg.git
-    version: 0.21.1
+    version: 0.26.0
 
   sqlite3:
     git: https://github.com/crystal-lang/crystal-sqlite3.git
-    version: 0.16.0
+    version: 0.19.0
 

shard.yml

@@ -1,5 +1,5 @@
 name: access_log
-version: 0.2.0
+version: 0.3.0
 
 authors:
   - f <f@sutty.nl>

src/access_log.cr

@@ -1,4 +1,6 @@
+require "file_utils"
 require "json"
+require "socket"
 require "sqlite3"
 require "pg"
 require "option_parser"
@@ -6,8 +8,10 @@ require "uuid"
 require "./models/access_log"
 require "./models/crawler"
 
-VERSION = "0.2.0"
+VERSION = "0.4.0"
 
+# Default socket location
+socket = "/tmp/access_log.socket"
 # The default database URI
 database = "sqlite3://./development.sqlite3"
 # Detect web crawlers
@@ -16,7 +20,7 @@ crawler = false
 crawlers = [] of Crawler
 # Fields in database
 # TODO: Obtain them from AccessLog
-fields = ["id", "remote_user", "host", "msec", "server_protocol", "request_method", "request_completion", "uri", "query_string", "status", "sent_http_content_type", "sent_http_content_encoding", "sent_http_etag", "sent_http_last_modified", "http_accept", "http_accept_encoding", "http_accept_language", "http_pragma", "http_cache_control", "http_if_none_match", "http_dnt", "http_user_agent", "http_origin", "http_referer", "request_time", "bytes_sent", "body_bytes_sent", "request_length", "http_connection", "pipe", "connection_requests", "geoip2_data_country_name", "geoip2_data_city_name", "ssl_server_name", "ssl_protocol", "ssl_early_data", "ssl_session_reused", "ssl_curves", "ssl_ciphers", "ssl_cipher", "sent_http_x_xss_protection", "sent_http_x_frame_options", "sent_http_x_content_type_options", "sent_http_strict_transport_security", "nginx_version", "pid", "crawler"]
+fields = ["id", "remote_user", "host", "msec", "server_protocol", "request_method", "request_completion", "uri", "request_uri", "query_string", "status", "sent_http_content_type", "sent_http_content_encoding", "sent_http_etag", "sent_http_last_modified", "http_accept", "http_accept_encoding", "http_accept_language", "http_pragma", "http_cache_control", "http_if_none_match", "http_dnt", "http_user_agent", "http_origin", "http_referer", "request_time", "bytes_sent", "body_bytes_sent", "request_length", "http_connection", "pipe", "connection_requests", "geoip2_data_country_name", "geoip2_data_city_name", "ssl_server_name", "ssl_protocol", "ssl_early_data", "ssl_session_reused", "ssl_curves", "ssl_ciphers", "ssl_cipher", "sent_http_x_xss_protection", "sent_http_x_frame_options", "sent_http_x_content_type_options", "sent_http_strict_transport_security", "nginx_version", "pid", "crawler"]
 # Params for the query
 params = [] of String
 
@@ -43,6 +47,10 @@ OptionParser.parse do |p|
     crawler = true
     crawlers = Array(Crawler).from_json File.read(c)
   end
+
+  p.on "-s /tmp/access_log.socket", "--socket /tmp/access_log.socket", "Listening socket" do |s|
+    socket = s
+  end
 end
 
 # Parameterize values according to database URI
@@ -65,20 +73,28 @@ if crawler
   crawler_re = Regex.union(crawlers.map { |c| c.pattern })
 end
 
-# Just exit
+server = Socket.unix(Socket::Type::DGRAM)
+server.bind Socket::UNIXAddress.new(socket)
+
 Signal::INT.trap do
+  server.close
+  FileUtils.rm(socket) if File.exists? socket
   exit
 end
 
-# Same
 Signal::KILL.trap do
+  server.close
+  FileUtils.rm(socket) if File.exists? socket
   exit
 end
 
 # Open the database and wait for JSONL input.
 db = DB.open database do |db|
-  while (json = gets)
+  while true
     begin
+      msg, _ = server.receive(1024 * 64) # 64K
+      _, _, _, _, json = msg.split(" ", 5)
+
       # Parse input
       access_log = AccessLog.from_json(json || "{}")
 
@@ -94,6 +110,7 @@ db = DB.open database do |db|
               access_log.request_method,
               access_log.request_completion,
               access_log.uri,
+              access_log.request_uri,
               access_log.query_string,
               access_log.status,
               access_log.sent_http_content_type,
@@ -136,6 +153,10 @@ db = DB.open database do |db|
 
     # Ignore parsing errors
     rescue JSON::ParseException
+    rescue IO::Error
+      server.close
+      FileUtils.rm(socket) if File.exists? socket
+      exit
     end
   end
 end

src/json/request_uri_converter.cr

@@ -0,0 +1,7 @@
+# Removes query string from request URI as we are already storing it in
+# query_string.
+module JSON::RequestUriConverter(Converter)
+  def self.from_json(value : JSON::PullParser)
+    value.read_string.split("?", 2).first
+  end
+end

src/models/access_log.cr

@@ -1,5 +1,6 @@
 require "json"
 require "../json/referrer_policy_converter"
+require "../json/request_uri_converter"
 
 class AccessLog
   include JSON::Serializable
@@ -11,6 +12,10 @@ class AccessLog
   property request_method : String
   property request_completion : String
   property uri : String
+
+  @[JSON::Field(converter: JSON::RequestUriConverter(String::Converter))]
+  property request_uri : String
+
   property query_string : String
   property status : Int32
   property sent_http_content_type : String
@@ -53,4 +58,3 @@ class AccessLog
   property nginx_version : String
   property pid : String
 end
-