v0.7.2

https://github.com/thegreenwebfoundation/co2.js/commit/b0a65fe073c8a4d7fc2adbafb1844e4cae3ee6c2

.gitlab-ci.yml

@@ -0,0 +1,60 @@
+stages:
+- "test"
+- "build"
+- "upload"
+- "release"
+variables:
+  RELEASE_DIRECTORY: "access-log-${CI_COMMIT_TAG}-linux-amd64"
+  RELEASE_TARBALL: "${RELEASE_DIRECTORY}.tar.gz"
+  PACKAGE_REGISTRY_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/access_log/${CI_COMMIT_TAG}/${RELEASE_TARBALL}"
+test:
+  stage: "test"
+  image: "crystallang/crystal:latest-alpine"
+  script:
+  - "shards install"
+  - "crystal spec"
+build:
+  rules:
+  - if: "$CI_COMMIT_TAG"
+  stage: "build"
+  image: "crystallang/crystal:latest-alpine"
+  cache:
+    paths:
+    - "lib/"
+  script:
+  - "apk add sqlite-static"
+  - "shards install"
+  - "crystal build --release --static src/access_log.cr"
+  - "crystal build --release --static src/ip2asn_server.cr"
+  - "strip --strip-all access_log ip2asn_server"
+  - "install -Dm 644 LICENSE ${RELEASE_DIRECTORY}/LICENSE"
+  - "install -Dm 755 access_log ${RELEASE_DIRECTORY}/access_log"
+  - "install -Dm 755 ip2asn_server ${RELEASE_DIRECTORY}/ip2asn_server"
+  - "tar -cf ${RELEASE_TARBALL} ${RELEASE_DIRECTORY}"
+  artifacts:
+    paths:
+    - "${RELEASE_TARBALL}"
+upload:
+  rules:
+  - if: "$CI_COMMIT_TAG"
+  stage: "upload"
+  image: "curlimages/curl:latest"
+  script:
+  - "curl --header \"JOB-TOKEN: ${CI_JOB_TOKEN}\" --upload-file ${RELEASE_TARBALL} ${PACKAGE_REGISTRY_URL}"
+release:
+  rules:
+  - if: "$CI_COMMIT_TAG"
+  stage: "release"
+  image: "registry.gitlab.com/gitlab-org/release-cli:latest"
+  script:
+  - "echo \"running release job\""
+  release:
+    name: "access_log $CI_COMMIT_TAG"
+    description: "$CI_COMMIT_MESSAGE"
+    tag_name: "$CI_COMMIT_TAG"
+    ref: "$CI_COMMIT_TAG"
+    assets:
+      links:
+      - name: "Static Linux binary"
+        url: "${PACKAGE_REGISTRY_URL}"
+        link_type: "package"

README.md

@@ -1,6 +1,6 @@
 # access_log
 
-Receives access logs from stdin in JSON format and stores them on
+Receives access logs on a UNIX socket in JSON format and stores them on
 a database.  It **intentionally** doesn't collect IP addresses.  It
 doesn't respect the Do Not Track (DNT) header though, because we're not
 collecting personally identifiable data.  Referrer collection is
@@ -16,6 +16,55 @@ configuration](https://0xacab.org/sutty/ansible-sutty/blob/master/templates/site
 
 It supports SQlite3 and PostgreSQL databases :)
 
+## Sustainable Web Design
+
+When enabled, you can track CO2 emissions using [Sustainable Web Design
+"Calculating Digital Emissions"
+method](https://sustainablewebdesign.org/calculating-digital-emissions/).
+The algorithm and data are based on
+[CO2.js](https://github.com/thegreenwebfoundation/co2.js).
+
+It follows the calculations with the added --optional-- feature of using
+the origin country of the visit for the "consumer device" segment.
+To enable this, see Nginx configuration.
+
+```bash
+# For a datacenter using renewable energy on Costa Rica
+access_log --swd --renewable --datacenter CR
+```
+
+### Average vs marginal intensity
+
+[CO2.js explains this
+better](https://developers.thegreenwebfoundation.org/co2js/data/).  In
+practice, using average intensity data will give lower results and
+mostly use the global intensity, since the data by country is missing
+most countries.
+
+`access_log` uses marginal data by default.
+
+### Update the data
+
+1. Go to the `data/output` directory on [co2.js
+   repository](https://github.com/thegreenwebfoundation/co2.js/tree/v0.16.2/data/output)
+   for the latest version released (in this example 0.16.2)
+
+2. Download the `average-intensities.json` file.
+
+3. Run `src/average_intensities_by_country.cr` with this file as stdin.
+
+```
+crystal run src/average_intensities_by_country.cr < average-intensities.json >> src/swd/average_intensity.cr
+```
+
+4. Modify `src/swd/average_intensity.cr` to fix the data.
+
+## Create database
+
+```bash
+sqlite3 access_log.sqlite3 < contrib/create.sql
+```
+
 ## Build
 
 Install zlib, sqlite3 and ssl development files (it varies between
@@ -37,58 +86,8 @@ make alpine-build
 
 ## Database configuration
 
-Create an `access_logs` database with the following schema:
-
-| Field                               | Type    | Reference                            | Index? |
-| -----                               | ----    | ---------                            | ------ |
-| id                                  | String  | UUID                                 | Unique |
-| host                                | String  | Host name                            | Yes    |
-| msec                                | Float   | Unix timestamp of visit              | ?      |
-| server_protocol                     | String  | HTTP/Version                         | ?      |
-| request_method                      | String  | GET/POST/etc.                        | ?      |
-| request_completion                  | String  | "OK"                                 | ?      |
-| uri                                 | String  | Request                              | True   |
-| query_string                        | String  | Arguments                            | ?      |
-| status                              | Integer | HTTP status                          | ?      |
-| sent_http_content_type              | String  | MIME type of response                | ?      |
-| sent_http_content_encoding          | String  | Compression                          | ?      |
-| sent_http_etag                      | String  | ETag header                          | ?      |
-| sent_http_last_modified             | String  | Last modified date                   | ?      |
-| http_accept                         | String  | MIME types requested                 | ?      |
-| http_accept_encoding                | String  | Compression accepted                 | ?      |
-| http_accept_language                | String  | Languages supported                  | ?      |
-| http_pragma                         | String  | Pragma header                        | ?      |
-| http_cache_control                  | String  | Cache requested                      | ?      |
-| http_if_none_match                  | String  | ETag requested                       | ?      |
-| http_dnt                            | String  | Do Not Track header                  | ?      |
-| http_user_agent                     | String  | User Agent                           | Yes    |
-| http_origin                         | String  | Request origin                       | Yes    |
-| http_referer                        | String  | Referer (see Referrer Policy)        | Yes    |
-| request_time                        | Float   | Request duration                     | ?      |
-| bytes_sent                          | Integer | Bytes sent                           | ?      |
-| body_bytes_sent                     | Integer | Bytes sent not including headers     | ?      |
-| request_length                      | Integer | Headers                              | ?      |
-| http_connection                     | String  | Connection status                    | ?      |
-| pipe                                | String  | Connection was multiplexed           | ?      |
-| connection_requests                 | Integer | Requests done on the same connection | ?      |
-| geoip2_data_country_name            | String  | Country according to GeoIP           | Yes    |
-| geoip2_data_city_name               | String  | City according to GeoIP              | Yes    |
-| ssl_server_name                     | String  | SNI                                  | ?      |
-| ssl_protocol                        | String  | SSL/TLS version used                 | ?      |
-| ssl_early_data                      | String  | TLSv1.3 early data used              | ?      |
-| ssl_session_reused                  | String  | TLS session reused                   | ?      |
-| ssl_curves                          | String  | Curves used                          | ?      |
-| ssl_ciphers                         | String  | Ciphers available                    | ?      |
-| ssl_cipher                          | String  | Cipher used                          | ?      |
-| sent_http_x_xss_protection          | String  | XSS Protection sent                  | ?      |
-| sent_http_x_frame_options           | String  | Frame protection sent                | ?      |
-| sent_http_x_content_type_options    | String  | Content protection sent              | ?      |
-| sent_http_strict_transport_security | String  | HSTS sent                            | ?      |
-| nginx_version                       | String  | Server version                       | ?      |
-| pid                                 | Integer | Server PID                           | ?      |
-| crawler                             | Boolean | Web crawler detected                 | ?      |
-| remote_user                         | String  | HTTP Basic auth user                 | ?      |
-
+Create an `access_logs` database with an schema similar to
+[create.sql](contrib/schema.sql)
 
 ## Nginx configuration
 
@@ -111,20 +110,53 @@ Configure Nginx to format access log as JSON.  You can configure
 `http_referer` (double and single "r" respectively, the second is a typo
 on the HTTP specification).
 
-Install `daemonize` and run `access_logd` to create `access.log` as
-a FIFO node, so Nginx writes to it and `access_log` can read from it.
+Install `daemonize` and run `access_logd`.  By default it creates a UNIX
+socket on `/tmp/access_log.socket` so Nginx writes can write to it using
+its [syslog support](https://nginx.org/en/docs/syslog.html).
+
 Check `/var/log/nginx/error.log` for debugging.
 
 `ACCESS_LOG_FLAGS` is the env variable to pass flags to `access_logd`.
 For a working example check our [Nginx
 container](https://0xacab.org/sutty/containers/nginx/).
 
-```json
+```
 log_format main escape=json '{"host":"$host","msec":$msec,"server_protocol":"$server_protocol","request_method":"$request_method","request_completion":"$request_completion","uri":"$uri","query_string":"$query_string","status":$status,"sent_http_content_type":"$sent_http_content_type","sent_http_content_encoding":"$sent_http_content_encoding","sent_http_etag":"$sent_http_etag","sent_http_last_modified":"$sent_http_last_modified","http_accept":"$http_accept","http_accept_encoding":"$http_accept_encoding","http_accept_language":"$http_accept_language","http_pragma":"$http_pragma","http_cache_control":"$http_cache_control","http_if_none_match":"$http_if_none_match","http_dnt":"$http_dnt","http_user_agent":"$http_user_agent","http_origin":"$http_origin","http_referer":{"origin":"$http_origin","referrer":"$http_referer","policy":"origin-when-cross-origin"},"request_time":$request_time,"bytes_sent":$bytes_sent,"body_bytes_sent":$body_bytes_sent,"request_length":$request_length,"http_connection":"$http_connection","pipe":"$pipe","connection_requests":$connection_requests,"geoip2_data_country_name":"$geoip2_data_country_name","geoip2_data_city_name":"$geoip2_data_city_name","ssl_server_name":"$ssl_server_name","ssl_protocol":"$ssl_protocol","ssl_early_data":"$ssl_early_data","ssl_session_reused":"$ssl_session_reused","ssl_curves":"$ssl_curves","ssl_ciphers":"$ssl_ciphers","ssl_cipher":"$ssl_cipher","sent_http_x_xss_protection":"$sent_http_x_xss_protection","sent_http_x_frame_options":"$sent_http_x_frame_options","sent_http_x_content_type_options":"$sent_http_x_content_type_options","sent_http_strict_transport_security":"$sent_http_strict_transport_security","nginx_version":"$nginx_version","pid":"$pid","remote_user":""}';
 
-access_log /var/log/nginx/access.log main;
+access_log syslog=unix:/tmp/access_log.socket,nohostname main;
 ```
 
+### Add origin country of visit to SWD
+
+Add a `$geoip2_data_country_iso_code` variable on Nginx and the
+corresponding variable to the JSON log format.
+
+```nginx
+geoip2 /usr/share/GeoIP/GeoLite2-Country.mmdb {
+  $geoip2_data_country_iso_code country iso_code;
+}
+
+log_format main escape=json '{"host":"$host","msec":$msec,"server_protocol":"$server_protocol","request_method":"$request_method","request_completion":"$request_completion","uri":"$uri","query_string":"$query_string","status":$status,"sent_http_content_type":"$sent_http_content_type","sent_http_content_encoding":"$sent_http_content_encoding","sent_http_etag":"$sent_http_etag","sent_http_last_modified":"$sent_http_last_modified","http_accept":"$http_accept","http_accept_encoding":"$http_accept_encoding","http_accept_language":"$http_accept_language","http_pragma":"$http_pragma","http_cache_control":"$http_cache_control","http_if_none_match":"$http_if_none_match","http_dnt":"$http_dnt","http_user_agent":"$http_user_agent","http_origin":"$http_origin","http_referer":{"origin":"$http_origin","referrer":"$http_referer","policy":"origin-when-cross-origin"},"request_time":$request_time,"bytes_sent":$bytes_sent,"body_bytes_sent":$body_bytes_sent,"request_length":$request_length,"http_connection":"$http_connection","pipe":"$pipe","connection_requests":$connection_requests,"geoip2_data_country_name":"$geoip2_data_country_name","geoip2_data_city_name":"$geoip2_data_city_name","ssl_server_name":"$ssl_server_name","ssl_protocol":"$ssl_protocol","ssl_early_data":"$ssl_early_data","ssl_session_reused":"$ssl_session_reused","ssl_curves":"$ssl_curves","ssl_ciphers":"$ssl_ciphers","ssl_cipher":"$ssl_cipher","sent_http_x_xss_protection":"$sent_http_x_xss_protection","sent_http_x_frame_options":"$sent_http_x_frame_options","sent_http_x_content_type_options":"$sent_http_x_content_type_options","sent_http_strict_transport_security":"$sent_http_strict_transport_security","nginx_version":"$nginx_version","pid":"$pid","remote_user":"","geoip2_data_country_iso_code":"$geoip2_data_country_iso_code"}';
+```
+
+Then run the program with the required flags enabled:
+
+```bash
+access_log --swd --device-country
+```
+
+## ASN database
+
+If you want to keep track of ASN for each visitor, for instance for
+grouping possible attacks or IA crawls, create a database based on
+<https://iptoasn.com/>:
+
+```bash
+./contrib/asn_database.sh
+```
+
+And start the server with the `--asn-database=` flag.
+
 ## Crawler user agents
 
 Download the [crawler user agents

access_logd

@@ -1,16 +1,11 @@
 #!/bin/sh
 
-# Fail if there's no options provided
+# Fail if there are no options provided
 test -n "$ACCESS_LOGS_FLAGS" || exit 1
 
-# Remove access.log and recreate it as a fifo
-rm -f /var/log/nginx/access.log /run/access_log.pid
-mkfifo /var/log/nginx/access.log
-chmod 755 /var/log/nginx/access.log
-
-# Reopen log file
-nginx -s reload
+# Give write access to Nginx
+umask 007
 
 # Read from fifo and load into database
 daemonize -p /run/access_logd.pid -u nobody \
-  /bin/sh -c "cat /var/log/nginx/access.log | access_log $ACCESS_LOGS_FLAGS"
+  /usr/bin/access_log $ACCESS_LOGS_FLAGS

contrib/asn.sql

@@ -0,0 +1,12 @@
+BEGIN;
+CREATE TABLE IF NOT EXISTS asn_import (range_start INTEGER, range_end INTEGER, id INTEGER, country STRING, description TEXT);
+CREATE TABLE IF NOT EXISTS asns (range_start INTEGER, range_end INTEGER, id INTEGER, country STRING, owner_id INTEGER);
+CREATE UNIQUE INDEX IF NOT EXISTS asn_import_range_id ON asn_import (range_start, range_end);
+CREATE UNIQUE INDEX IF NOT EXISTS asns_range_id ON asns (range_start, range_end);
+CREATE INDEX IF NOT EXISTS asns_id_idx ON asns (id);
+CREATE INDEX IF NOT EXISTS asns_owner_id_idx ON asns (owner_id);
+CREATE TABLE IF NOT EXISTS owners (id INTEGER PRIMARY KEY, description TEXT);
+CREATE TABLE IF NOT EXISTS status (id INTEGER PRIMARY KEY, blocked BOOLEAN DEFAULT FALSE);
+CREATE TABLE IF NOT EXISTS owner_status (id INTEGER PRIMARY KEY, blocked BOOLEAN DEFAULT FALSE);
+DELETE FROM asn_import;
+COMMIT;

contrib/asn_2.sql

@@ -0,0 +1,12 @@
+BEGIN;
+INSERT INTO owners (description) SELECT DISTINCT asn_import.description FROM asn_import LEFT JOIN owners ON asn_import.description = owners.description WHERE owners.description IS NULL;
+INSERT INTO asns (range_start, range_end, id, country, owner_id)
+  SELECT asn_import.range_start, asn_import.range_end, asn_import.id, asn_import.country, owners.id
+  FROM asn_import
+  LEFT JOIN owners
+  ON owners.description = asn_import.description
+ON CONFLICT DO UPDATE SET id = excluded.id, country = excluded.country, owner_id = excluded.owner_id;
+DROP TABLE asn_import;
+DROP INDEX asns_range_id;
+COMMIT;
+VACUUM;

contrib/asn_database.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -e
+
+cr="$(dirname "$(readlink -f "$0")")"
+db="${1:-$PWD/asn.sqlite3}"
+cat "$cr/asn.sql" | sqlite3 "$db"
+wget https://iptoasn.com/data/ip2asn-v4-u32.tsv.gz -O - | gunzip | sqlite3 -tabs "$db" ".import '|cat -' asn_import"
+cat "$cr/asn_2.sql" | sqlite3 "$db"
+
+echo "ASN database created/updated at $db"

contrib/create.sql

@@ -0,0 +1,76 @@
+PRAGMA journal_mode=WAL;
+
+CREATE TABLE IF NOT EXISTS "access_logs" (
+ "id" NOT NULL PRIMARY KEY,
+ "host" varchar DEFAULT NULL,
+ "msec" float DEFAULT NULL,
+ "server_protocol" varchar DEFAULT NULL,
+ "request_method" varchar DEFAULT NULL,
+ "request_completion" varchar DEFAULT NULL,
+ "uri" varchar DEFAULT NULL,
+ "query_string" varchar DEFAULT NULL,
+ "status" integer DEFAULT NULL,
+ "sent_http_content_type" varchar DEFAULT NULL,
+ "sent_http_content_encoding" varchar DEFAULT NULL,
+ "sent_http_etag" varchar DEFAULT NULL,
+ "sent_http_last_modified" varchar DEFAULT NULL,
+ "http_accept" varchar DEFAULT NULL,
+ "http_accept_encoding" varchar DEFAULT NULL,
+ "http_accept_language" varchar DEFAULT NULL,
+ "http_pragma" varchar DEFAULT NULL,
+ "http_cache_control" varchar DEFAULT NULL,
+ "http_if_none_match" varchar DEFAULT NULL,
+ "http_dnt" varchar DEFAULT NULL,
+ "http_user_agent" varchar DEFAULT NULL,
+ "http_origin" varchar DEFAULT NULL,
+ "request_time" float DEFAULT NULL,
+ "bytes_sent" integer DEFAULT NULL,
+ "body_bytes_sent" integer DEFAULT NULL,
+ "request_length" integer DEFAULT NULL,
+ "http_connection" varchar DEFAULT NULL,
+ "pipe" varchar DEFAULT NULL,
+ "connection_requests" integer DEFAULT NULL,
+ "geoip2_data_country_name" varchar DEFAULT NULL,
+ "geoip2_data_city_name" varchar DEFAULT NULL,
+ "ssl_server_name" varchar DEFAULT NULL,
+ "ssl_protocol" varchar DEFAULT NULL,
+ "ssl_early_data" varchar DEFAULT NULL,
+ "ssl_session_reused" varchar DEFAULT NULL,
+ "ssl_curves" varchar DEFAULT NULL,
+ "ssl_ciphers" varchar DEFAULT NULL,
+ "ssl_cipher" varchar DEFAULT NULL,
+ "sent_http_x_xss_protection" varchar DEFAULT NULL,
+ "sent_http_x_frame_options" varchar DEFAULT NULL,
+ "sent_http_x_content_type_options" varchar DEFAULT NULL,
+ "sent_http_strict_transport_security" varchar DEFAULT NULL,
+ "nginx_version" varchar DEFAULT NULL,
+ "pid" integer DEFAULT NULL,
+ "remote_user" varchar DEFAULT NULL,
+ "crawler" boolean DEFAULT 0,
+ "http_referer" varchar DEFAULT NULL,
+ "request_uri" varchar DEFAULT NULL,
+ "datacenter_co2" float DEFAULT NULL,
+ "network_co2" float DEFAULT NULL,
+ "consumer_device_co2" float DEFAULT NULL,
+ "production_co2" float DEFAULT NULL,
+ "total_co2" float DEFAULT NULL,
+ "node" varchar DEFAULT NULL,
+ "asn" integer DEFAULT NULL,
+ "http3" varchar(2) DEFAULT NULL,
+ "http_sec_fetch_mode" varchar DEFAULT NULL,
+ "http_sec_fetch_dest" varchar DEFAULT NULL,
+ "http_sec_fetch_site" varchar DEFAULT NULL,
+ "http_sec_fetch_user" varchar DEFAULT NULL,
+ "http_sec_purpose" varchar DEFAULT NULL,
+ "limit_req_status" varchar DEFAULT NULL,
+ "limit_conn_status" varchar DEFAULT NULL
+);
+
+CREATE INDEX IF NOT EXISTS "index_access_logs_on_host" ON "access_logs" ("host");
+CREATE INDEX IF NOT EXISTS "index_access_logs_on_uri" ON "access_logs" ("uri");
+CREATE INDEX IF NOT EXISTS "index_access_logs_on_request_uri" ON "access_logs" ("request_uri");
+CREATE INDEX IF NOT EXISTS "index_access_logs_on_status" ON "access_logs" ("status");
+CREATE INDEX IF NOT EXISTS "index_access_logs_on_http_user_agent" ON "access_logs" ("http_user_agent");
+CREATE INDEX IF NOT EXISTS "index_access_logs_on_geoip2_data_country_name" ON "access_logs" ("geoip2_data_country_name");
+CREATE INDEX IF NOT EXISTS "index_access_logs_on_geoip2_data_city_name" ON "access_logs" ("geoip2_data_city_name");
+CREATE INDEX IF NOT EXISTS "index_access_logs_on_http_origin" ON "access_logs" ("http_origin");

contrib/ip2asn_nginx.conf

@@ -0,0 +1,19 @@
+keyval_zone_redis zone=ip2asn hostname=::1;
+keyval "$remote_addr:id" $asn zone=ip2asn;
+keyval "$remote_addr:blocked" $asn_blocked zone=ip2asn;
+
+limit_conn_zone $asn zone=asn_conn:10m;
+limit_req_zone $asn zone=asn_req:10m rate=10r/s;
+
+server {
+  server_name _;
+
+  listen 80 default_server;
+
+  if ($asn_blocked) {
+    return 444;
+  }
+
+  limit_conn asn_conn 1;
+  limit_req zone=asn_req nodelay;
+}

shard.lock

@@ -1,14 +1,18 @@
 version: 2.0
 shards:
+  blank:
+    git: https://github.com/kostya/blank.git
+    version: 0.2.0
+
   db:
     git: https://github.com/crystal-lang/crystal-db.git
-    version: 0.9.0
+    version: 0.11.0
 
   pg:
     git: https://github.com/will/crystal-pg.git
-    version: 0.21.1
+    version: 0.26.0
 
   sqlite3:
     git: https://github.com/crystal-lang/crystal-sqlite3.git
-    version: 0.16.0
+    version: 0.19.0
 

shard.yml

@@ -1,19 +1,16 @@
-name: access_log
-version: 0.2.0
-
+name: "access_log"
+version: "0.7.2"
 authors:
-  - f <f@sutty.nl>
-
+- "f <f@sutty.nl>"
 targets:
   access_log:
-    main: src/access_log.cr
-
-crystal: 0.35.1
-
-license: MIT-Antifa
-
+    main: "src/access_log.cr"
+crystal: "0.35.1"
+license: "MIT-Antifa"
 dependencies:
   pg:
-    github: will/crystal-pg
+    github: "will/crystal-pg"
   sqlite3:
-    github: crystal-lang/crystal-sqlite3
+    github: "crystal-lang/crystal-sqlite3"
+  blank:
+    github: "kostya/blank"

spec/swd_spec.cr

@@ -0,0 +1,141 @@
+require "spec"
+require "../src/swd"
+
+# XXX: Why was it a float number originally?
+AVERAGE_WEBSITE_IN_BYTES = 2257715;
+
+# XXX: The be_close values were taken from the co2.js tests
+describe SWD do
+  swd = SWD.new(AVERAGE_WEBSITE_IN_BYTES)
+
+  describe "#new" do
+    it "can be initialized" do
+      swd.should(be_a(SWD))
+    end
+  end
+
+  describe "#transfered_bytes_to_gb" do
+    it "should convert bytes to gigabytes" do
+      swd.transfered_bytes_to_gb.should(eq 0.002257715)
+    end
+  end
+
+  describe "#energy_usage" do
+    it "should convert gb to energy" do
+      swd.energy_usage.should(eq 0.0027873024691358024)
+    end
+  end
+
+  describe "#consumer_device_energy" do
+    it "should extract the consumer device energy usage" do
+      swd.consumer_device_energy.should(eq 0.0014493972839506174)
+      swd.consumer_device_energy.should(be_close 0.00095095, 8)
+    end
+  end
+
+  describe "#production_energy" do
+    it "should extract the production energy usage" do
+      swd.production_energy.should(eq 0.0005295874691358025)
+      swd.production_energy.should(be_close 0.0003475, 7)
+    end
+  end
+
+  describe "#network_energy" do
+    it "should extract the network energy usage" do
+      swd.network_energy.should(eq 0.0003902223456790124)
+      swd.network_energy.should(be_close 0.00025602, 7)
+    end
+  end
+
+  describe "#datacenter_energy" do
+    it "should extract the datacenter energy usage" do
+      swd.datacenter_energy.should(eq 0.00041809537037037033)
+      swd.datacenter_energy.should(be_close 0.00027431, 8)
+    end
+  end
+
+  describe "#total_energy" do
+    it "should extract the total energy usage" do
+      swd.total_energy.should(eq 0.0027873024691358024)
+      swd.total_energy.should(be_close 0.00182874, 7)
+    end
+  end
+
+  describe "#datacenter_carbon_intensity" do
+    it "should be a global value by default" do
+      swd.datacenter_carbon_intensity.should(eq SWD::GLOBAL_GRID_INTENSITY)
+    end
+
+    it "should be renewable when specified" do
+      SWD.new(AVERAGE_WEBSITE_IN_BYTES, true).datacenter_carbon_intensity.should(eq SWD::RENEWABLES_GRID_INTENSITY)
+    end
+
+    it "can be initialized with a country iso code" do
+      SWD.new(AVERAGE_WEBSITE_IN_BYTES, false, "AR").datacenter_carbon_intensity.should(eq SWD::IntensityData.by_iso("AR"))
+    end
+  end
+
+  describe "#device_carbon_intensity" do
+    it "should be a global value by default" do
+      swd.device_carbon_intensity.should(eq SWD::GLOBAL_GRID_INTENSITY)
+    end
+
+    it "should be global even when datacenter uses renewable energy" do
+      SWD.new(AVERAGE_WEBSITE_IN_BYTES, true).device_carbon_intensity.should(eq SWD::GLOBAL_GRID_INTENSITY)
+    end
+
+    it "can be initialized with a country iso code" do
+      SWD.new(AVERAGE_WEBSITE_IN_BYTES, false, nil, "AR").device_carbon_intensity.should(eq SWD::IntensityData.by_iso("AR"))
+    end
+  end
+
+  describe "#network_carbon_intensity" do
+    it "should be a global value by default" do
+      swd.network_carbon_intensity.should(eq SWD::GLOBAL_GRID_INTENSITY)
+    end
+
+    it "should be global even when datacenter uses renewable energy" do
+      SWD.new(AVERAGE_WEBSITE_IN_BYTES, true).network_carbon_intensity.should(eq SWD::GLOBAL_GRID_INTENSITY)
+    end
+  end
+
+  describe "#global_emissions" do
+    it "should be a global value by default" do
+      swd.global_emissions.should(eq SWD::GLOBAL_GRID_INTENSITY)
+    end
+
+    it "should be global even when datacenter uses renewable energy" do
+      SWD.new(AVERAGE_WEBSITE_IN_BYTES, true).global_emissions.should(eq SWD::GLOBAL_GRID_INTENSITY)
+    end
+  end
+
+  describe "#consumer_device_co2" do
+    it "should extract the consumer device co2 emissions" do
+      swd.consumer_device_co2.should(eq 0.6854924454444445)
+    end
+  end
+
+  describe "#production_co2" do
+    it "should extract the production co2 emissions" do
+      swd.production_co2.should(eq 0.2504683935277778)
+    end
+  end
+
+  describe "#network_co2" do
+    it "should extract the network co2 emissions" do
+      swd.network_co2.should(eq 0.1845556583888889)
+    end
+  end
+
+  describe "#datacenter_co2" do
+    it "should extract the datacenter co2 emissions" do
+      swd.datacenter_co2.should(eq 0.19773820541666665)
+    end
+  end
+
+  describe "#total_co2" do
+    it "should extract the total co2 emissions" do
+      swd.total_co2.should(eq 1.3182547027777778)
+    end
+  end
+end

src/access_log.cr

@@ -1,24 +1,48 @@
+require "log"
+require "file_utils"
 require "json"
+require "socket"
 require "sqlite3"
 require "pg"
 require "option_parser"
 require "uuid"
+require "system"
 require "./models/access_log"
 require "./models/crawler"
+require "./models/ai_bot"
+require "./swd"
+require "./asn"
+require "./presence"
 
-VERSION = "0.2.0"
+VERSION = "0.7.2"
 
+Log.setup_from_env
+
+# Default socket location
+socket = "/tmp/access_log.socket"
 # The default database URI
 database = "sqlite3://./development.sqlite3"
+# ASN database
+asn_database = ""
 # Detect web crawlers
 crawler = false
+ai_bot = false
 # Parse the crawlers repository
 crawlers = [] of Crawler
+ai_bots = AiBots.new
 # Fields in database
 # TODO: Obtain them from AccessLog
-fields = ["id", "remote_user", "host", "msec", "server_protocol", "request_method", "request_completion", "uri", "query_string", "status", "sent_http_content_type", "sent_http_content_encoding", "sent_http_etag", "sent_http_last_modified", "http_accept", "http_accept_encoding", "http_accept_language", "http_pragma", "http_cache_control", "http_if_none_match", "http_dnt", "http_user_agent", "http_origin", "http_referer", "request_time", "bytes_sent", "body_bytes_sent", "request_length", "http_connection", "pipe", "connection_requests", "geoip2_data_country_name", "geoip2_data_city_name", "ssl_server_name", "ssl_protocol", "ssl_early_data", "ssl_session_reused", "ssl_curves", "ssl_ciphers", "ssl_cipher", "sent_http_x_xss_protection", "sent_http_x_frame_options", "sent_http_x_content_type_options", "sent_http_strict_transport_security", "nginx_version", "pid", "crawler"]
+fields = %w[id remote_user host msec server_protocol request_method request_completion uri request_uri query_string status sent_http_content_type sent_http_content_encoding sent_http_etag sent_http_last_modified http_accept http_accept_encoding http_accept_language http_pragma http_cache_control http_if_none_match http_dnt http_user_agent http_origin http_referer request_time bytes_sent body_bytes_sent request_length http_connection pipe connection_requests geoip2_data_country_name geoip2_data_city_name ssl_server_name ssl_protocol ssl_early_data ssl_session_reused ssl_curves ssl_ciphers ssl_cipher sent_http_x_xss_protection sent_http_x_frame_options sent_http_x_content_type_options sent_http_strict_transport_security nginx_version pid crawler datacenter_co2 network_co2 consumer_device_co2 production_co2 total_co2 node asn http3 http_sec_fetch_mode http_sec_fetch_dest http_sec_fetch_site http_sec_fetch_user http_sec_purpose limit_req_status limit_conn_status ai_bot]
 # Params for the query
 params = [] of String
+# SWD
+swd = false
+# Datacenter
+datacenter : String? = nil
+renewable = false
+device_country = false
+intensity = :marginal
+node = false
 
 # Parse CLI flags
 OptionParser.parse do |p|
@@ -39,14 +63,52 @@ OptionParser.parse do |p|
     database = d
   end
 
+  p.on "-I DATABASE", "--asn-database=DATABASE", "ASN Database URI" do |d|
+    asn_database = d
+  end
+
   p.on "-c crawler-user-agents.json", "--crawlers crawler-user-agents.json", "Crawlers repository" do |c|
     crawler = true
     crawlers = Array(Crawler).from_json File.read(c)
   end
+
+  p.on "-i ai-robots.json", "--ai-bots ai-robots.json", "AI bots repository" do |c|
+    crawler = true
+    ai_bot = true
+    ai_bots = AiBots.from_json File.read(c)
+  end
+
+  p.on "-s /tmp/access_log.socket", "--socket /tmp/access_log.socket", "Listening socket" do |s|
+    socket = s
+  end
+
+  p.on "-S", "--swd", "Enable Sustainable Web Design CO2 emission tracking" do
+    swd = true
+  end
+
+  p.on "-D ISO2", "--datacenter ISO2", "Datacenter ISO 2-letter country code" do |d|
+    datacenter = d
+  end
+
+  p.on "-R", "--renewable", "Datacenter uses renewable energy" do
+    renewable = true
+  end
+
+  p.on "-C", "--device-country", "Take country of visit for SWD" do
+    device_country = true
+  end
+
+  p.on "-A", "--average-intensity", "Use average intensity figures rather than marginal" do
+    intensity = :average
+  end
+
+  p.on "-n", "--node", "Send node hostname" do
+    node = true
+  end
 end
 
 # Parameterize values according to database URI
-if database.starts_with? "postgresql://"
+if database.starts_with?("postgresql://") || database.starts_with?("postgres://")
   params = fields.map_with_index do |_, i|
     "$#{i + 1}"
   end
@@ -63,79 +125,134 @@ query = "insert into access_logs (#{fields.join(",")}) values (#{params.join(","
 # regex.
 if crawler
   crawler_re = Regex.union(crawlers.map { |c| c.pattern })
+
+  if ai_bots
+    ai_bot_re = Regex.union(ai_bots.keys)
+    crawler_re = Regex.union(crawler_re, ai_bot_re)
+  end
+end
+
+def remove_socket!(socket)
+  FileUtils.rm(socket) if File.exists? socket
+end
+
+remove_socket!(socket)
+
+asn = ASN.new(asn_database) unless asn_database.empty?
+server = Socket.unix(Socket::Type::DGRAM)
+server.bind Socket::UNIXAddress.new(socket)
+
+def close_server!(server, asn)
+  server.try(&.close)
+  asn.try(&.close)
 end
 
-# Just exit
 Signal::INT.trap do
+  remove_socket!(socket)
+  close_server!(server, asn)
   exit
 end
 
-# Same
 Signal::KILL.trap do
+  remove_socket!(socket)
+  close_server!(server, asn)
   exit
 end
 
+MAX_SIZE = 1024 * 64 # 64K
+
 # Open the database and wait for JSONL input.
-db = DB.open database do |db|
-  while (json = gets)
+DB.open database do |db|
+  while true
     begin
+      msg, _ = server.receive(MAX_SIZE)
+      # TODO: is this the best method?
+      _, json = msg.split("{", 2)
+      json = "{#{json}"
+
       # Parse input
       access_log = AccessLog.from_json(json || "{}")
+      d = device_country ? access_log.geoip2_data_country_iso_code : nil
+      s = SWD.new(access_log.bytes_sent, renewable, datacenter, d, intensity) if swd
+      i = access_log.remote_addr
+      a = asn.try(&.query(i)) if i
 
       # Execute query, detect crawler if enabled.
       # TODO: Does Crystal support splats?  It does but we need to cast
       # AccessLog as a NamedTuple.
       db.exec query,
               UUID.random.to_s,
-              access_log.remote_user,
-              access_log.host,
-              access_log.msec,
-              access_log.server_protocol,
-              access_log.request_method,
-              access_log.request_completion,
-              access_log.uri,
-              access_log.query_string,
+              access_log.remote_user.presence,
+              access_log.host.presence,
+              access_log.msec.presence,
+              access_log.server_protocol.presence,
+              access_log.request_method.presence,
+              access_log.request_completion.presence,
+              access_log.uri.presence,
+              access_log.request_uri.presence,
+              access_log.query_string.presence,
               access_log.status,
-              access_log.sent_http_content_type,
-              access_log.sent_http_content_encoding,
-              access_log.sent_http_etag,
-              access_log.sent_http_last_modified,
-              access_log.http_accept,
-              access_log.http_accept_encoding,
-              access_log.http_accept_language,
-              access_log.http_pragma,
-              access_log.http_cache_control,
-              access_log.http_if_none_match,
-              access_log.http_dnt,
-              access_log.http_user_agent,
-              access_log.http_origin,
-              access_log.http_referer,
+              access_log.sent_http_content_type.presence,
+              access_log.sent_http_content_encoding.presence,
+              access_log.sent_http_etag.presence,
+              access_log.sent_http_last_modified.presence,
+              access_log.http_accept.presence,
+              access_log.http_accept_encoding.presence,
+              access_log.http_accept_language.presence,
+              access_log.http_pragma.presence,
+              access_log.http_cache_control.presence,
+              access_log.http_if_none_match.presence,
+              access_log.http_dnt.presence,
+              access_log.http_user_agent.presence,
+              access_log.http_origin.presence,
+              access_log.http_referer.presence,
               access_log.request_time,
               access_log.bytes_sent,
               access_log.body_bytes_sent,
               access_log.request_length,
-              access_log.http_connection,
-              access_log.pipe,
+              access_log.http_connection.presence,
+              access_log.pipe.presence,
               access_log.connection_requests,
-              access_log.geoip2_data_country_name,
-              access_log.geoip2_data_city_name,
-              access_log.ssl_server_name,
-              access_log.ssl_protocol,
-              access_log.ssl_early_data,
-              access_log.ssl_session_reused,
-              access_log.ssl_curves,
-              access_log.ssl_ciphers,
-              access_log.ssl_cipher,
-              access_log.sent_http_x_xss_protection,
-              access_log.sent_http_x_frame_options,
-              access_log.sent_http_x_content_type_options,
-              access_log.sent_http_strict_transport_security,
-              access_log.nginx_version,
-              access_log.pid,
-              (crawler ? !!(crawler_re =~ access_log.http_user_agent) : false)
+              access_log.geoip2_data_country_name.presence,
+              access_log.geoip2_data_city_name.presence,
+              access_log.ssl_server_name.presence,
+              access_log.ssl_protocol.presence,
+              access_log.ssl_early_data.presence,
+              access_log.ssl_session_reused.presence,
+              access_log.ssl_curves.presence,
+              access_log.ssl_ciphers.presence,
+              access_log.ssl_cipher.presence,
+              access_log.sent_http_x_xss_protection.presence,
+              access_log.sent_http_x_frame_options.presence,
+              access_log.sent_http_x_content_type_options.presence,
+              access_log.sent_http_strict_transport_security.presence,
+              access_log.nginx_version.presence,
+              access_log.pid.presence,
+              (crawler ? !!(crawler_re =~ access_log.http_user_agent) : false),
+              (swd ? s.try(&.datacenter_co2) : nil),
+              (swd ? s.try(&.network_co2) : nil),
+              (swd ? s.try(&.consumer_device_co2) : nil),
+              (swd ? s.try(&.production_co2) : nil),
+              (swd ? s.try(&.total_co2) : nil),
+              (node ? System.hostname : nil),
+              a.try(&.id),
+              access_log.http3.presence,
+              access_log.http_sec_fetch_mode.presence,
+              access_log.http_sec_fetch_dest.presence,
+              access_log.http_sec_fetch_site.presence,
+              access_log.http_sec_fetch_user.presence,
+              access_log.http_sec_purpose.presence,
+              access_log.limit_req_status.presence,
+              access_log.limit_conn_status.presence,
+              (ai_bot ? !!(ai_bot_re =~ access_log.http_user_agent) : false)
 
     # Ignore parsing errors
-    rescue JSON::ParseException
+    rescue e : JSON::ParseException
+      Log.warn &.emit("Parse exception", error: e.message)
+    rescue IO::Error
+      remove_socket!(socket)
+      close_server!(server, asn)
+      exit
     end
   end
 end

src/asn.cr

@@ -0,0 +1,55 @@
+require "db"
+require "sqlite3"
+
+class AsnResult
+  include DB::Serializable
+
+  property id : Int64?
+  property country : String?
+  # TODO: Deprecate?
+  property description : String?
+  property owner_id : Int64?
+  property blocked : Bool?
+end
+
+class ASN
+  QUERY = "select asns.id, country, owners.description, owner_id from asns left join owners on owners.id = asns.owner_id where ? between range_start and range_end limit 1;"
+  STATUS = "select blocked from status where id = ? limit 1;"
+  OWNER_STATUS = "select blocked from owner_status where id = ? limit 1;"
+
+  property db : DB::Database
+
+  def initialize(database : String)
+    @db = DB.open(database)
+    @cache = Hash(UInt32, AsnResult).new
+  end
+
+  # Binary representation of an IPv4 address
+  def to_binary(address : String) : UInt32
+    (address.split(".").map(&.to_u32).reduce(0.to_u32) { |t, v| (t << 8) + v })
+  end
+
+  # Lookups an address and returns the ASN for it
+  def query(address : String)
+    query(to_binary(address))
+  end
+
+  def query(binary_address : UInt32) : AsnResult?
+    (@cache[binary_address] ||= @db.query_one(QUERY, binary_address.to_i64, &.read(AsnResult))).tap do |result|
+      result.blocked   = rescue_query { @db.query_one(STATUS, result.id, &.read(Bool)) }
+      result.blocked ||= rescue_query { @db.query_one(OWNER_STATUS, result.owner_id, &.read(Bool)) }
+    end
+  rescue DB::NoResultsError
+    nil
+  end
+
+  def close
+    @db.close
+  end
+
+  def rescue_query(&)
+    yield
+  rescue DB::NoResultsError
+    nil
+  end
+end

src/average_intensities_by_country.cr

@@ -0,0 +1,34 @@
+require "json"
+require "csv"
+
+csv = CSV.new(File.open("./countries_codes_and_coordinates.csv"), headers: true)
+countries = Hash(String, String).new
+
+while csv.next
+  countries[csv["Alpha-3 code"]] = csv["Alpha-2 code"]
+end
+
+average_intensities = Hash(String, Float64).new
+
+JSON.parse(ARGF.gets_to_end).as_h.each do |country, data|
+  if data["country_code"].as_s.empty?
+    STDERR.puts "not ok - Skipping #{country} because of missing ISO code"
+    next
+  end
+
+  unless countries.has_key? data["country_code"].as_s
+    STDERR.puts "not ok - Skipping #{country} because of missing ISO2 code"
+    next
+  end
+
+  STDERR.puts "ok - #{country}"
+
+  average_intensities[countries[data["country_code"].as_s]] =
+    begin
+      data["emissions_intensity_gco2_per_kwh"].as_f
+    rescue TypeCastError
+      data["emissions_intensity_gco2_per_kwh"].as_i.to_f
+    end
+end
+
+puts average_intensities.to_json

src/ip2asn_server.cr

@@ -0,0 +1,93 @@
+# This is a very minimal Redis server that works with Nginx keyval
+# module.  When set like this:
+#
+# keyval_zone_redis zone=redis hostname=::1;
+# keyval "$remote_addr:id" $asn_id zone=redis;
+#
+# We can query the ASN id from a database and use it on Nginx config to
+# rate limit entire subnets.
+require "log"
+require "option_parser"
+require "./asn"
+require "./redis_server"
+
+Log.setup_from_env
+
+VERSION = "0.2.0"
+ASN_ID = "id"
+ASN_DESCRIPTION = "description"
+ASN_OWNER_ID = "owner_id"
+ASN_COUNTRY = "country"
+ASN_BLOCKED = "blocked"
+
+asn_database = "sqlite3://./asn.sqlite3"
+interface = "localhost"
+port = 6379
+
+OptionParser.parse do |p|
+  p.banner = "IP to ASN Redis server"
+
+  p.on "-v", "--version", "Show version" do
+    puts VERSION
+    exit
+  end
+
+  p.on "-h", "--help", "Show help" do
+    puts p
+    exit
+  end
+
+  p.on "-d #{asn_database}", "--asn-database=#{asn_database}", "ASN Database URI" do |d|
+    asn_database = d
+  end
+
+  p.on "-i #{interface}", "--interface=#{interface}", "Listening interface" do |i|
+    interface = i
+  end
+
+  p.on "-p #{port}", "--port=#{port}", "Listening port" do |p|
+    port = p.to_i
+  end
+end
+
+asn = ASN.new(asn_database)
+
+# The client can be a long lived connection sending commands. We only
+# answer to GET commands.
+RedisServer.new(interface, port) do |client, _, key|
+  begin
+    _, ip, field = key.split(":", 3)
+  rescue
+    Log.warn &.emit("Error parsing key", key: key)
+    next
+  end
+
+  # This is cached in memory
+  result = asn.query(ip)
+
+  next if result.nil?
+
+  # XXX: Are we going to use the other fields?
+  response =
+    case field
+    when ASN_ID then result.try(&.id).try(&.to_s)
+    when ASN_OWNER_ID then result.try(&.owner_id).try(&.to_s)
+    when ASN_COUNTRY then result.try(&.country)
+    when ASN_DESCRIPTION then result.try(&.description)
+    when ASN_BLOCKED then
+      if result.try(&.blocked)
+        "1"
+      else
+        "0"
+      end
+    else nil
+    end
+
+  if response.nil?
+    RedisServer.null(client)
+  else
+    RedisServer.bulk_string(client, response)
+  end
+end
+
+asn.close

src/json/request_uri_converter.cr

@@ -0,0 +1,7 @@
+# Removes query string from request URI as we are already storing it in
+# query_string.
+module JSON::RequestUriConverter(Converter)
+  def self.from_json(value : JSON::PullParser)
+    value.read_string.split("?", 2).first
+  end
+end

src/marginal_intensities_by_country.cr

@@ -0,0 +1,14 @@
+require "json"
+
+marginal_intensities = Hash(String, Float32).new
+
+JSON.parse(ARGF.gets_to_end).as_h.each do |country, data|
+  if data["country_code_iso_2"].as_s.empty?
+    STDERR.puts "not ok - Skipping #{country} because of missing ISO code"
+    next
+  end
+
+  marginal_intensities[data["country_code_iso_2"].as_s] = data["Operating Margin Grid Emission"].as_s.to_f32
+end
+
+puts marginal_intensities.to_json

src/models/access_log.cr

@@ -1,5 +1,6 @@
 require "json"
 require "../json/referrer_policy_converter"
+require "../json/request_uri_converter"
 
 class AccessLog
   include JSON::Serializable
@@ -11,6 +12,10 @@ class AccessLog
   property request_method : String
   property request_completion : String
   property uri : String
+
+  @[JSON::Field(converter: JSON::RequestUriConverter(String::Converter))]
+  property request_uri : String
+
   property query_string : String
   property status : Int32
   property sent_http_content_type : String
@@ -38,6 +43,7 @@ class AccessLog
   property pipe : String
   property connection_requests : Int32
   property geoip2_data_country_name : String
+  property geoip2_data_country_iso_code : String?
   property geoip2_data_city_name : String
   property ssl_server_name : String
   property ssl_protocol : String
@@ -52,5 +58,13 @@ class AccessLog
   property sent_http_strict_transport_security : String
   property nginx_version : String
   property pid : String
+  property remote_addr : String?
+  property http3 : String?
+  property http_sec_fetch_mode : String?
+  property http_sec_fetch_dest : String?
+  property http_sec_fetch_site : String?
+  property http_sec_fetch_user : String?
+  property http_sec_purpose : String?
+  property limit_req_status : String?
+  property limit_conn_status : String?
 end
-

src/models/ai_bot.cr

@@ -0,0 +1,13 @@
+require "json"
+
+class AiBot
+  include JSON::Serializable
+
+  property operator : String
+  property respect : String
+  property function : String
+  property frequency : String
+  property description : String
+end
+
+AiBots = Hash(String, AiBot)

src/presence.cr

@@ -0,0 +1,9 @@
+require "blank"
+
+class Object
+  def presence
+    return nil if blank?
+
+    self
+  end
+end

src/redis_server.cr

@@ -0,0 +1,64 @@
+require "log"
+require "socket"
+
+class RedisServer
+  BULK_STRING = "$"
+  CRLF = "\r\n"
+  NOT_IMPLEMENTED = "-Not implemented#{CRLF}"
+  QUIT_COMMAND = "quit"
+  GET_COMMAND = "get"
+  NULL = "_#{CRLF}"
+
+  property server : TCPServer
+
+  def initialize(interface : String = "localhost", port : Int = 6379, &block : TCPSocket, String, String -> _)
+    @server = TCPServer.new(interface, port)
+
+    while (client = server.accept?)
+      spawn handle_client(client, block)
+    end
+  ensure
+    server.close
+  end
+
+  def handle_client(client : TCPSocket, block : Proc)
+    next_is_key = false
+    last_command = ""
+
+    while (line = client.gets)
+      # We're not a compliant RESP
+      next if line.starts_with?("*")
+      next if line.starts_with?("$")
+
+      # Execute the query when we get the key
+      if next_is_key
+        next_is_key = false
+
+        # Do something with the key
+        block.call(client, last_command, line)
+      # First we get the Redis command and we only support a small subset.
+      else
+        case (last_command = line.downcase)
+        when QUIT_COMMAND then client.close
+        when GET_COMMAND then next_is_key = true
+        else
+          client.print NOT_IMPLEMENTED
+
+          raise last_command
+        end
+      end
+    end
+  rescue ex
+    Log.warn &.emit("Error", exception: ex.class.name, error: ex.message)
+  ensure
+    client.close
+  end
+
+  def self.bulk_string(client : TCPSocket, string : String)
+    client.print(BULK_STRING, string.size, CRLF, string, CRLF)
+  end
+
+  def self.null(client : TCPSocket)
+    client.print(NULL)
+  end
+end

src/swd.cr

@@ -0,0 +1,127 @@
+# This is an adaptation of the Sustainable Web Design algorithm to
+# calculate CO2 emissions per visit.
+#
+# In this case if we know the country of origin of the visit we can
+# adapt the device emissions to the country, same as the datacenter
+# variable on the original proposal.
+#
+# Also the CO2 per visit is adapted to the actual cache, since we know
+# the status code.  304 visits are cached so the energy usage is much
+# smaller and we don't need to make assumptions.
+#
+# @see https://sustainablewebdesign.org/calculating-digital-emissions/
+
+require "./swd/intensity_data"
+
+class SWD
+  getter bytes : Int8 | Int16 | Int32 | Int64 | UInt8 | UInt16 | UInt32 | UInt64
+  getter renewable : Bool
+  getter datacenter_iso_code : String?
+  getter consumer_device_iso_code : String?
+  getter intensity_calculator : Symbol
+
+  # @see {co2.js/blob/main/src/constants/index.js}
+  GIGABYTE = 1000.0 * 1000.0 * 1000.0
+  KWH_PER_GB = 0.81
+  END_USER_DEVICE_ENERGY = 0.52
+  NETWORK_ENERGY = 0.14
+  DATACENTER_ENERGY = 0.15
+  PRODUCTION_ENERGY = 0.19
+  GLOBAL_GRID_INTENSITY = 472.95
+  RENEWABLES_GRID_INTENSITY = 50.0
+
+  @transfered_bytes_to_gb : Float32? | Float64? = nil
+  @energy_usage : Float32? | Float64? = nil
+  @consumer_device_energy : Float32? | Float64? = nil
+  @network_energy : Float32? | Float64? = nil
+  @production_energy : Float32? | Float64? = nil
+  @datacenter_energy : Float32? | Float64? = nil
+  @total_energy : Float32? | Float64? = nil
+  @consumer_device_co2 : Float32? | Float64? = nil
+  @network_co2 : Float32? | Float64? = nil
+  @production_co2 : Float32? | Float64? = nil
+  @datacenter_co2 : Float32? | Float64? = nil
+  @total_co2 : Float32? | Float64? = nil
+
+  def initialize(@bytes, @renewable = false, @datacenter_iso_code = nil, @consumer_device_iso_code = nil, @intensity_calculator = :marginal); end
+
+  def transfered_bytes_to_gb
+    @transfered_bytes_to_gb ||= @bytes / GIGABYTE
+  end
+
+  def energy_usage
+    @energy_usage ||= transfered_bytes_to_gb / KWH_PER_GB
+  end
+
+  def consumer_device_energy
+    @consumer_device_energy ||= energy_usage * END_USER_DEVICE_ENERGY
+  end
+
+  def network_energy
+    @network_energy ||= energy_usage * NETWORK_ENERGY
+  end
+
+  def production_energy
+    @production_energy ||= energy_usage * PRODUCTION_ENERGY
+  end
+
+  def total_energy
+    @total_energy ||= production_energy + network_energy + datacenter_energy + consumer_device_energy
+  end
+
+  def datacenter_energy
+    @datacenter_energy ||= energy_usage * DATACENTER_ENERGY
+  end
+
+  # Returns the carbon intensity depending on renewable status and
+  # location, by default uses global grid intensity.
+  def datacenter_carbon_intensity : Float
+    renewable ? RENEWABLES_GRID_INTENSITY : (localized_datacenter_intensity || GLOBAL_GRID_INTENSITY)
+  end
+
+  # Returns the carbon intensity of user devices depending on location,
+  # by default uses global grid intensity
+  def device_carbon_intensity : Float
+    localized_consumer_device_intensity || GLOBAL_GRID_INTENSITY
+  end
+
+  def network_carbon_intensity : Float
+    GLOBAL_GRID_INTENSITY
+  end
+
+  def global_emissions : Float
+    GLOBAL_GRID_INTENSITY
+  end
+
+  def datacenter_co2
+    @datacenter_co2 ||= datacenter_carbon_intensity * datacenter_energy
+  end
+
+  def consumer_device_co2
+    @consumer_device_co2 ||= device_carbon_intensity * consumer_device_energy
+  end
+
+  def network_co2
+    @network_co2 ||= network_carbon_intensity * network_energy
+  end
+
+  def production_co2
+    @production_co2 ||= production_energy * global_emissions
+  end
+
+  def total_co2
+    @total_co2 ||= production_co2 + network_co2 + datacenter_co2 + consumer_device_co2
+  end
+
+  def localized_datacenter_intensity : Float64?
+    return unless datacenter_iso_code
+
+    SWD::IntensityData.by_iso(datacenter_iso_code || "", intensity_calculator)
+  end
+
+  def localized_consumer_device_intensity : Float64?
+    return unless consumer_device_iso_code
+
+    SWD::IntensityData.by_iso(consumer_device_iso_code || "", intensity_calculator)
+  end
+end

src/swd/intensity_data.cr

@@ -0,0 +1,238 @@
+class SWD
+  module IntensityData
+    # @see {co2.js/data/output/average-intensities.json}
+    AVERAGE_INTENSITY_BY_ISO_CODE = {
+      "AE": 467.51,
+      "AF": 123.71,
+      "AG": 611.11,
+      "AL": 24.42,
+      "AM": 243.52,
+      "AO": 167.22,
+      "AR": 344.83,
+      "AS": 647.06,
+      "AT": 102.62,
+      "AU": 553.76,
+      "AW": 550.0,
+      "AZ": 632.89,
+      "BA": 638.05,
+      "BB": 600.0,
+      "BD": 694.63,
+      "BE": 117.58,
+      "BF": 554.91,
+      "BG": 264.21,
+      "BH": 902.41,
+      "BI": 230.77,
+      "BJ": 590.0,
+      "BN": 892.67,
+      "BO": 468.02,
+      "BR": 106.06,
+      "BS": 653.66,
+      "BT": 24.19,
+      "BW": 849.42,
+      "BY": 323.63,
+      "BZ": 155.56,
+      "CA": 184.99,
+      "CD": 27.04,
+      "CF": 0.0,
+      "CG": 713.73,
+      "CH": 36.72,
+      "CI": 393.53,
+      "CK": 250.0,
+      "CL": 265.52,
+      "CM": 285.71,
+      "CN": 557.5,
+      "CO": 285.8,
+      "CR": 63.01,
+      "CU": 638.98,
+      "CV": 480.0,
+      "CY": 512.24,
+      "CZ": 413.86,
+      "DE": 342.06,
+      "DJ": 450.0,
+      "DK": 143.3,
+      "DM": 600.0,
+      "DO": 565.97,
+      "DZ": 633.65,
+      "EC": 209.7,
+      "EE": 341.02,
+      "EG": 574.5,
+      "ER": 590.91,
+      "ES": 146.15,
+      "ET": 23.55,
+      "FI": 72.25,
+      "FJ": 278.26,
+      "FK": 1000.0,
+      "FO": 354.17,
+      "FR": 44.18,
+      "GA": 429.47,
+      "GB": 215.79,
+      "GD": 666.67,
+      "GE": 143.06,
+      "GF": 204.08,
+      "GH": 452.86,
+      "GL": 111.11,
+      "GM": 666.67,
+      "GN": 182.72,
+      "GP": 493.9,
+      "GQ": 605.1,
+      "GR": 319.76,
+      "GT": 272.66,
+      "GU": 611.11,
+      "GW": 625.0,
+      "GY": 634.33,
+      "HK": 681.99,
+      "HN": 289.5,
+      "HR": 174.48,
+      "HT": 534.65,
+      "HU": 182.82,
+      "ID": 682.43,
+      "IE": 279.79,
+      "IL": 567.26,
+      "IN": 708.96,
+      "IQ": 689.4,
+      "IR": 648.68,
+      "IS": 28.33,
+      "IT": 287.75,
+      "JM": 561.25,
+      "JO": 539.21,
+      "JP": 483.73,
+      "KE": 84.83,
+      "KG": 152.65,
+      "KH": 497.46,
+      "KI": 500.0,
+      "KM": 642.86,
+      "KN": 636.36,
+      "KP": 344.26,
+      "KR": 415.65,
+      "KW": 637.24,
+      "KY": 642.86,
+      "KZ": 801.79,
+      "LA": 232.12,
+      "LB": 369.47,
+      "LC": 650.0,
+      "LK": 509.78,
+      "LR": 435.9,
+      "LS": 20.83,
+      "LT": 139.34,
+      "LU": 134.62,
+      "LV": 136.22,
+      "LY": 830.53,
+      "MA": 577.65,
+      "MD": 629.56,
+      "ME": 413.51,
+      "MG": 477.27,
+      "MK": 568.97,
+      "ML": 394.5,
+      "MM": 569.69,
+      "MN": 784.01,
+      "MO": 448.98,
+      "MQ": 516.78,
+      "MR": 481.71,
+      "MS": 1000.0,
+      "MT": 484.16,
+      "MU": 633.03,
+      "MV": 611.77,
+      "MW": 54.65,
+      "MX": 483.14,
+      "MY": 604.43,
+      "MZ": 127.81,
+      "NA": 47.62,
+      "NC": 585.76,
+      "NE": 687.5,
+      "NG": 507.85,
+      "NI": 288.33,
+      "NL": 252.7,
+      "NO": 30.75,
+      "NP": 23.36,
+      "NR": 750.0,
+      "NZ": 120.11,
+      "OM": 545.33,
+      "PA": 258.74,
+      "PE": 263.27,
+      "PF": 436.62,
+      "PG": 513.74,
+      "PH": 613.38,
+      "PK": 398.61,
+      "PL": 614.98,
+      "PM": 600.0,
+      "PR": 664.53,
+      "PS": 460.78,
+      "PT": 111.8,
+      "PY": 24.86,
+      "QA": 602.83,
+      "RE": 525.22,
+      "RO": 245.55,
+      "RS": 670.8,
+      "RU": 446.17,
+      "RW": 301.89,
+      "SA": 691.95,
+      "SB": 636.36,
+      "SC": 571.43,
+      "SD": 214.33,
+      "SE": 35.89,
+      "SG": 498.74,
+      "SI": 227.11,
+      "SK": 96.49,
+      "SL": 47.62,
+      "SN": 535.4,
+      "SO": 523.81,
+      "SR": 383.18,
+      "SS": 610.17,
+      "ST": 555.56,
+      "SV": 99.29,
+      "SY": 682.27,
+      "SZ": 142.86,
+      "TC": 653.85,
+      "TD": 615.39,
+      "TG": 478.26,
+      "TH": 555.43,
+      "TJ": 98.7,
+      "TM": 1306.3,
+      "TN": 560.25,
+      "TO": 571.43,
+      "TR": 469.7,
+      "TT": 682.11,
+      "TW": 635.15,
+      "TZ": 371.59,
+      "UA": 250.47,
+      "UG": 57.39,
+      "US": 383.55,
+      "UY": 96.7,
+      "UZ": 1121.18,
+      "VC": 600.0,
+      "VE": 180.25,
+      "VG": 647.06,
+      "VI": 641.79,
+      "VN": 486.13,
+      "VU": 500.0,
+      "WS": 400.0,
+      "XK": 958.72,
+      "YE": 586.32,
+      "ZA": 713.9,
+      "ZM": 111.0,
+      "ZW": 298.44
+    }
+
+    # @see {co2.js/data/output/marginal-intensities-2021.json}
+    MARGINAL_INTENSITY_BY_ISO_CODE = {"AF": 414.0, "AL": 0.0, "DZ": 528.0, "AS": 753.0, "AD": 188.0, "AO": 1476.0, "AI": 753.0, "AG": 753.0, "AR": 478.0, "AM": 390.0, "AW": 753.0, "AU": 808.0, "AT": 242.0, "AZ": 534.0, "BS": 753.0, "BH": 726.0, "BD": 528.0, "BB": 749.0, "BY": 400.0, "BE": 252.0, "BZ": 403.0, "BJ": 745.0, "BM": 753.0, "BT": 0.0, "BO": 604.0, "BQ": 753.0, "BA": 1197.0, "BW": 1486.0, "BR": 284.0, "VG": 753.0, "BN": 681.0, "BG": 911.0, "BF": 753.0, "BI": 414.0, "KH": 1046.0, "CM": 659.0, "CA": 372.0, "KY": 753.0, "CV": 753.0, "CY": 751.0, "CF": 188.0, "TD": 753.0, "CL": 657.0, "CN": 899.0, "CO": 410.0, "KM": 753.0, "CD": 0.0, "CG": 659.0, "CK": 753.0, "CR": 108.0, "CI": 466.0, "HR": 294.0, "CU": 559.0, "CW": 876.0, "CZ": 902.0, "DK": 362.0, "DJ": 753.0, "DM": 753.0, "DO": 601.0, "EC": 560.0, "EG": 554.0, "SV": 547.0, "GQ": 632.0, "ER": 915.0, "EE": 1057.0, "SZ": 0.0, "ET": 0.0, "FK": 753.0, "FO": 753.0, "FJ": 640.0, "FI": 267.0, "FR": 158.0, "GF": 423.0, "PF": 753.0, "GA": 946.0, "GM": 753.0, "GE": 289.0, "DE": 650.0, "GH": 495.0, "GI": 779.0, "GR": 507.0, "GL": 264.0, "GD": 753.0, "GP": 753.0, "GU": 753.0, "GT": 798.0, "GN": 753.0, "GW": 753.0, "GY": 847.0, "HT": 1048.0, "HN": 662.0, "HU": 296.0, "IS": 0.0, "IN": 951.0, "ID": 783.0, "IR": 592.0, "IQ": 1080.0, "IE": 380.0, "IM": 436.0, "IL": 394.0, "IT": 414.0, "JM": 711.0, "JP": 471.0, "JO": 529.0, "KZ": 797.0, "KE": 574.0, "KI": 753.0, "KP": 754.0, "KR": 555.0, "XK": 1145.0, "KW": 675.0, "KG": 217.0, "LA": 1069.0, "LV": 240.0, "LB": 794.0, "LS": 0.0, "LR": 677.0, "LY": 668.0, "LI": 151.0, "LT": 211.0, "LU": 220.0, "MG": 876.0, "MW": 489.0, "MY": 551.0, "MV": 753.0, "ML": 1076.0, "MT": 520.0, "MH": 753.0, "MQ": 753.0, "MR": 753.0, "MU": 700.0, "YT": 753.0, "MX": 531.0, "FM": 753.0, "MD": 541.0, "MC": 158.0, "MN": 1366.0, "ME": 899.0, "MS": 753.0, "MA": 729.0, "MZ": 234.0, "MM": 719.0, "NA": 355.0, "NR": 753.0, "NP": 0.0, "NL": 326.0, "NC": 779.0, "NZ": 246.0, "NI": 675.0, "NE": 772.0, "NG": 526.0, "NU": 753.0, "MK": 851.0, "MP": 753.0, "NO": 47.0, "OM": 479.0, "PK": 592.0, "PW": 753.0, "PS": 719.0, "PA": 477.0, "PG": 597.0, "PY": 0.0, "PE": 473.0, "PH": 672.0, "PL": 828.0, "PT": 389.0, "PR": 596.0, "QA": 503.0, "RE": 772.0, "RO": 489.0, "RU": 476.0, "RW": 712.0, "SH": 753.0, "KN": 753.0, "LC": 753.0, "MF": 753.0, "PM": 753.0, "VC": 753.0, "WS": 753.0, "SM": 414.0, "ST": 753.0, "SA": 592.0, "SN": 870.0, "RS": 1086.0, "SC": 753.0, "SL": 489.0, "SG": 379.0, "SX": 753.0, "SK": 332.0, "SI": 620.0, "SB": 753.0, "SO": 753.0, "ZA": 1070.0, "SS": 890.0, "ES": 402.0, "LK": 731.0, "SD": 736.0, "SR": 1029.0, "SE": 68.0, "CH": 48.0, "SY": 713.0, "TW": 484.0, "TJ": 255.0, "TZ": 531.0, "TH": 450.0, "TL": 753.0, "TG": 859.0, "TO": 753.0, "TT": 559.0, "TN": 468.0, "TR": 376.0, "TM": 927.0, "TC": 753.0, "TV": 753.0, "UG": 279.0, "UA": 768.0, "AE": 556.0, "GB": 380.0, "US": 416.0, "UY": 174.0, "UZ": 612.0, "VU": 753.0, "VE": 711.0, "VN": 560.0, "VI": 650.0, "YE": 807.0, "ZM": 416.0, "ZW": 1575.0}
+
+    # Return the marginal intensity for a country when provided an ISO
+    # code
+    def self.marginal_by_iso(iso : String) : Float64?
+      MARGINAL_INTENSITY_BY_ISO_CODE[iso] if MARGINAL_INTENSITY_BY_ISO_CODE.has_key? iso
+    end
+
+    # Return the average intensity for a country when provided an ISO
+    # code
+    def self.average_by_iso(iso : String) : Float64?
+      AVERAGE_INTENSITY_BY_ISO_CODE[iso] if AVERAGE_INTENSITY_BY_ISO_CODE.has_key? iso
+    end
+
+    def self.by_iso(iso : String, type : Symbol = :marginal) : Float64?
+      case type
+      when :average then average_by_iso(iso)
+      else marginal_by_iso(iso)
+      end
+    end
+  end
+end