v0.5.5

swd #3

See merge request sutty/access_log!5

.gitlab-ci.yml

@@ -1,4 +1,5 @@
 stages:
+- "test"
 - "build"
 - "upload"
 - "release"
@@ -6,11 +7,17 @@ variables:
   RELEASE_DIRECTORY: "access-log-${CI_COMMIT_TAG}-linux-amd64"
   RELEASE_TARBALL: "${RELEASE_DIRECTORY}.tar.gz"
   PACKAGE_REGISTRY_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/access_log/${CI_COMMIT_TAG}/${RELEASE_TARBALL}"
-build:
-  stage: "build"
+test:
+  stage: "test"
   image: "crystallang/crystal:latest-alpine"
+  script:
+  - "shards install"
+  - "crystal spec"
+build:
   rules:
   - if: "$CI_COMMIT_TAG"
+  stage: "build"
+  image: "crystallang/crystal:latest-alpine"
   cache:
     paths:
     - "lib/"
@@ -19,23 +26,33 @@ build:
   - "shards install"
   - "crystal build --release --static src/access_log.cr"
   - "strip --strip-all access_log"
-  - "mkdir -p ${RELEASE_DIRECTORY}"
-  - "mv access_log LICENSE ${RELEASE_DIRECTORY}"
-  - "tar -czf ${RELEASE_TARBALL} ${RELEASE_DIRECTORY}"
+  - "install -Dm 644 LICENSE ${RELEASE_DIRECTORY}/LICENSE"
+  - "install -Dm 755 access_log ${RELEASE_DIRECTORY}/access_log"
+  - "tar -cf ${RELEASE_TARBALL} ${RELEASE_DIRECTORY}"
   artifacts:
     paths:
     - "${RELEASE_TARBALL}"
 upload:
-  stage: "upload"
-  image: "curlimages/curl:latest"
   rules:
   - if: "$CI_COMMIT_TAG"
+  stage: "upload"
+  image: "curlimages/curl:latest"
   script:
   - "curl --header \"JOB-TOKEN: ${CI_JOB_TOKEN}\" --upload-file ${RELEASE_TARBALL} ${PACKAGE_REGISTRY_URL}"
 release:
-  stage: "release"
-  image: "registry.gitlab.com/gitlab-org/release-cli:latest"
   rules:
   - if: "$CI_COMMIT_TAG"
+  stage: "release"
+  image: "registry.gitlab.com/gitlab-org/release-cli:latest"
   script:
-    - "release-cli create --name \"Release ${CI_COMMIT_TAG}\" --tag-name ${CI_COMMIT_TAG} --assets-link \"{\\\"name\\\":\\\"${RELEASE_TARBALL}\\\",\\\"url\\\":\\\"${PACKAGE_REGISTRY_URL}\\\",\\\"link_type\\\":\\\"package\\\"}\""
+  - "echo \"running release job\""
+  release:
+    name: "access_log $CI_COMMIT_TAG"
+    description: "$CI_COMMIT_MESSAGE"
+    tag_name: "$CI_COMMIT_TAG"
+    ref: "$CI_COMMIT_TAG"
+    assets:
+      links:
+      - name: "Static Linux binary"
+        url: "${PACKAGE_REGISTRY_URL}"
+        link_type: "package"

README.md

@@ -16,6 +16,39 @@ configuration](https://0xacab.org/sutty/ansible-sutty/blob/master/templates/site
 
 It supports SQlite3 and PostgreSQL databases :)
 
+## Sustainable Web Design
+
+When enabled, you can track CO2 emissions using [Sustainable Web Design
+"Calculating Digital Emissions"
+method](https://sustainablewebdesign.org/calculating-digital-emissions/).
+The algorithm and data are based on
+[CO2.js](https://github.com/thegreenwebfoundation/co2.js).
+
+It follows the calculations with the added --optional-- feature of using
+the origin country of the visit for the "consumer device" segment.
+To enable this, see Nginx configuration.
+
+```bash
+# For a datacenter using renewable energy on Costa Rica
+access_log --swd --renewable --datacenter CR
+```
+
+### Average vs marginal intensity
+
+[CO2.js explains this
+better](https://developers.thegreenwebfoundation.org/co2js/data/).  In
+practice, using average intensity data will give lower results and
+mostly use the global intensity, since the data by country is missing
+most countries.
+
+`access_log` uses marginal data by default.
+
+## Create database
+
+```bash
+sqlite3 access_log.sqlite3 < contrib/create.sql
+```
+
 ## Build
 
 Install zlib, sqlite3 and ssl development files (it varies between
@@ -121,12 +154,31 @@ Check `/var/log/nginx/error.log` for debugging.
 For a working example check our [Nginx
 container](https://0xacab.org/sutty/containers/nginx/).
 
-```json
+```
 log_format main escape=json '{"host":"$host","msec":$msec,"server_protocol":"$server_protocol","request_method":"$request_method","request_completion":"$request_completion","uri":"$uri","query_string":"$query_string","status":$status,"sent_http_content_type":"$sent_http_content_type","sent_http_content_encoding":"$sent_http_content_encoding","sent_http_etag":"$sent_http_etag","sent_http_last_modified":"$sent_http_last_modified","http_accept":"$http_accept","http_accept_encoding":"$http_accept_encoding","http_accept_language":"$http_accept_language","http_pragma":"$http_pragma","http_cache_control":"$http_cache_control","http_if_none_match":"$http_if_none_match","http_dnt":"$http_dnt","http_user_agent":"$http_user_agent","http_origin":"$http_origin","http_referer":{"origin":"$http_origin","referrer":"$http_referer","policy":"origin-when-cross-origin"},"request_time":$request_time,"bytes_sent":$bytes_sent,"body_bytes_sent":$body_bytes_sent,"request_length":$request_length,"http_connection":"$http_connection","pipe":"$pipe","connection_requests":$connection_requests,"geoip2_data_country_name":"$geoip2_data_country_name","geoip2_data_city_name":"$geoip2_data_city_name","ssl_server_name":"$ssl_server_name","ssl_protocol":"$ssl_protocol","ssl_early_data":"$ssl_early_data","ssl_session_reused":"$ssl_session_reused","ssl_curves":"$ssl_curves","ssl_ciphers":"$ssl_ciphers","ssl_cipher":"$ssl_cipher","sent_http_x_xss_protection":"$sent_http_x_xss_protection","sent_http_x_frame_options":"$sent_http_x_frame_options","sent_http_x_content_type_options":"$sent_http_x_content_type_options","sent_http_strict_transport_security":"$sent_http_strict_transport_security","nginx_version":"$nginx_version","pid":"$pid","remote_user":""}';
 
 access_log syslog=unix:/tmp/access_log.socket,nohostname main;
 ```
 
+### Add origin country of visit to SWD
+
+Add a `$geoip2_data_country_iso_code` variable on Nginx and the
+corresponding variable to the JSON log format.
+
+```nginx
+geoip2 /usr/share/GeoIP/GeoLite2-Country.mmdb {
+  $geoip2_data_country_iso_code country iso_code;
+}
+
+log_format main escape=json '{"host":"$host","msec":$msec,"server_protocol":"$server_protocol","request_method":"$request_method","request_completion":"$request_completion","uri":"$uri","query_string":"$query_string","status":$status,"sent_http_content_type":"$sent_http_content_type","sent_http_content_encoding":"$sent_http_content_encoding","sent_http_etag":"$sent_http_etag","sent_http_last_modified":"$sent_http_last_modified","http_accept":"$http_accept","http_accept_encoding":"$http_accept_encoding","http_accept_language":"$http_accept_language","http_pragma":"$http_pragma","http_cache_control":"$http_cache_control","http_if_none_match":"$http_if_none_match","http_dnt":"$http_dnt","http_user_agent":"$http_user_agent","http_origin":"$http_origin","http_referer":{"origin":"$http_origin","referrer":"$http_referer","policy":"origin-when-cross-origin"},"request_time":$request_time,"bytes_sent":$bytes_sent,"body_bytes_sent":$body_bytes_sent,"request_length":$request_length,"http_connection":"$http_connection","pipe":"$pipe","connection_requests":$connection_requests,"geoip2_data_country_name":"$geoip2_data_country_name","geoip2_data_city_name":"$geoip2_data_city_name","ssl_server_name":"$ssl_server_name","ssl_protocol":"$ssl_protocol","ssl_early_data":"$ssl_early_data","ssl_session_reused":"$ssl_session_reused","ssl_curves":"$ssl_curves","ssl_ciphers":"$ssl_ciphers","ssl_cipher":"$ssl_cipher","sent_http_x_xss_protection":"$sent_http_x_xss_protection","sent_http_x_frame_options":"$sent_http_x_frame_options","sent_http_x_content_type_options":"$sent_http_x_content_type_options","sent_http_strict_transport_security":"$sent_http_strict_transport_security","nginx_version":"$nginx_version","pid":"$pid","remote_user":"","geoip2_data_country_iso_code":"$geoip2_data_country_iso_code"}';
+```
+
+Then run the program with the required flags enabled:
+
+```bash
+access_log --swd --device-country
+```
+
 ## Crawler user agents
 
 Download the [crawler user agents

access_logd

@@ -7,5 +7,5 @@ test -n "$ACCESS_LOGS_FLAGS" || exit 1
 umask 007
 
 # Read from fifo and load into database
-daemonize -p /run/access_logd.pid -u nobody -g www-data \
+daemonize -p /run/access_logd.pid -u nobody \
   /usr/bin/access_log $ACCESS_LOGS_FLAGS

contrib/create.sql

@@ -0,0 +1,66 @@
+PRAGMA journal_mode=WAL;
+
+CREATE TABLE IF NOT EXISTS "access_logs" (
+ "id" NOT NULL PRIMARY KEY,
+ "host" varchar DEFAULT NULL,
+ "msec" float DEFAULT NULL,
+ "server_protocol" varchar DEFAULT NULL,
+ "request_method" varchar DEFAULT NULL,
+ "request_completion" varchar DEFAULT NULL,
+ "uri" varchar DEFAULT NULL,
+ "query_string" varchar DEFAULT NULL,
+ "status" integer DEFAULT NULL,
+ "sent_http_content_type" varchar DEFAULT NULL,
+ "sent_http_content_encoding" varchar DEFAULT NULL,
+ "sent_http_etag" varchar DEFAULT NULL,
+ "sent_http_last_modified" varchar DEFAULT NULL,
+ "http_accept" varchar DEFAULT NULL,
+ "http_accept_encoding" varchar DEFAULT NULL,
+ "http_accept_language" varchar DEFAULT NULL,
+ "http_pragma" varchar DEFAULT NULL,
+ "http_cache_control" varchar DEFAULT NULL,
+ "http_if_none_match" varchar DEFAULT NULL,
+ "http_dnt" varchar DEFAULT NULL,
+ "http_user_agent" varchar DEFAULT NULL,
+ "http_origin" varchar DEFAULT NULL,
+ "request_time" float DEFAULT NULL,
+ "bytes_sent" integer DEFAULT NULL,
+ "body_bytes_sent" integer DEFAULT NULL,
+ "request_length" integer DEFAULT NULL,
+ "http_connection" varchar DEFAULT NULL,
+ "pipe" varchar DEFAULT NULL,
+ "connection_requests" integer DEFAULT NULL,
+ "geoip2_data_country_name" varchar DEFAULT NULL,
+ "geoip2_data_city_name" varchar DEFAULT NULL,
+ "ssl_server_name" varchar DEFAULT NULL,
+ "ssl_protocol" varchar DEFAULT NULL,
+ "ssl_early_data" varchar DEFAULT NULL,
+ "ssl_session_reused" varchar DEFAULT NULL,
+ "ssl_curves" varchar DEFAULT NULL,
+ "ssl_ciphers" varchar DEFAULT NULL,
+ "ssl_cipher" varchar DEFAULT NULL,
+ "sent_http_x_xss_protection" varchar DEFAULT NULL,
+ "sent_http_x_frame_options" varchar DEFAULT NULL,
+ "sent_http_x_content_type_options" varchar DEFAULT NULL,
+ "sent_http_strict_transport_security" varchar DEFAULT NULL,
+ "nginx_version" varchar DEFAULT NULL,
+ "pid" integer DEFAULT NULL,
+ "remote_user" varchar DEFAULT NULL,
+ "crawler" boolean DEFAULT 0,
+ "http_referer" varchar DEFAULT NULL,
+ "request_uri" varchar DEFAULT NULL,
+ "datacenter_co2" float DEFAULT NULL,
+ "network_co2" float DEFAULT NULL,
+ "consumer_device_co2" float DEFAULT NULL,
+ "production_co2" float DEFAULT NULL,
+ "total_co2" float DEFAULT NULL
+);
+
+CREATE INDEX IF NOT EXISTS "index_access_logs_on_host" ON "access_logs" ("host");
+CREATE INDEX IF NOT EXISTS "index_access_logs_on_uri" ON "access_logs" ("uri");
+CREATE INDEX IF NOT EXISTS "index_access_logs_on_request_uri" ON "access_logs" ("request_uri");
+CREATE INDEX IF NOT EXISTS "index_access_logs_on_status" ON "access_logs" ("status");
+CREATE INDEX IF NOT EXISTS "index_access_logs_on_http_user_agent" ON "access_logs" ("http_user_agent");
+CREATE INDEX IF NOT EXISTS "index_access_logs_on_geoip2_data_country_name" ON "access_logs" ("geoip2_data_country_name");
+CREATE INDEX IF NOT EXISTS "index_access_logs_on_geoip2_data_city_name" ON "access_logs" ("geoip2_data_city_name");
+CREATE INDEX IF NOT EXISTS "index_access_logs_on_http_origin" ON "access_logs" ("http_origin");

shard.yml

@@ -1,19 +1,14 @@
-name: access_log
-version: 0.3.0
-
+name: "access_log"
+version: "0.5.5"
 authors:
-  - f <f@sutty.nl>
-
+- "f <f@sutty.nl>"
 targets:
   access_log:
-    main: src/access_log.cr
-
-crystal: 0.35.1
-
-license: MIT-Antifa
-
+    main: "src/access_log.cr"
+crystal: "0.35.1"
+license: "MIT-Antifa"
 dependencies:
   pg:
-    github: will/crystal-pg
+    github: "will/crystal-pg"
   sqlite3:
-    github: crystal-lang/crystal-sqlite3
+    github: "crystal-lang/crystal-sqlite3"

spec/swd_spec.cr

@@ -0,0 +1,141 @@
+require "spec"
+require "../src/swd"
+
+# XXX: Why was it a float number originally?
+AVERAGE_WEBSITE_IN_BYTES = 2257715;
+
+# XXX: The be_close values were taken from the co2.js tests
+describe SWD do
+  swd = SWD.new(AVERAGE_WEBSITE_IN_BYTES)
+
+  describe "#new" do
+    it "can be initialized" do
+      swd.should(be_a(SWD))
+    end
+  end
+
+  describe "#transfered_bytes_to_gb" do
+    it "should convert bytes to gigabytes" do
+      swd.transfered_bytes_to_gb.should(eq 0.002257715)
+    end
+  end
+
+  describe "#energy_usage" do
+    it "should convert gb to energy" do
+      swd.energy_usage.should(eq 0.0027873024691358024)
+    end
+  end
+
+  describe "#consumer_device_energy" do
+    it "should extract the consumer device energy usage" do
+      swd.consumer_device_energy.should(eq 0.0014493972839506174)
+      swd.consumer_device_energy.should(be_close 0.00095095, 8)
+    end
+  end
+
+  describe "#production_energy" do
+    it "should extract the production energy usage" do
+      swd.production_energy.should(eq 0.0005295874691358025)
+      swd.production_energy.should(be_close 0.0003475, 7)
+    end
+  end
+
+  describe "#network_energy" do
+    it "should extract the network energy usage" do
+      swd.network_energy.should(eq 0.0003902223456790124)
+      swd.network_energy.should(be_close 0.00025602, 7)
+    end
+  end
+
+  describe "#datacenter_energy" do
+    it "should extract the datacenter energy usage" do
+      swd.datacenter_energy.should(eq 0.00041809537037037033)
+      swd.datacenter_energy.should(be_close 0.00027431, 8)
+    end
+  end
+
+  describe "#total_energy" do
+    it "should extract the total energy usage" do
+      swd.total_energy.should(eq 0.0027873024691358024)
+      swd.total_energy.should(be_close 0.00182874, 7)
+    end
+  end
+
+  describe "#datacenter_carbon_intensity" do
+    it "should be a global value by default" do
+      swd.datacenter_carbon_intensity.should(eq SWD::GLOBAL_GRID_INTENSITY)
+    end
+
+    it "should be renewable when specified" do
+      SWD.new(AVERAGE_WEBSITE_IN_BYTES, true).datacenter_carbon_intensity.should(eq SWD::RENEWABLES_GRID_INTENSITY)
+    end
+
+    it "can be initialized with a country iso code" do
+      SWD.new(AVERAGE_WEBSITE_IN_BYTES, false, "AR").datacenter_carbon_intensity.should(eq SWD::IntensityData.by_iso("AR"))
+    end
+  end
+
+  describe "#device_carbon_intensity" do
+    it "should be a global value by default" do
+      swd.device_carbon_intensity.should(eq SWD::GLOBAL_GRID_INTENSITY)
+    end
+
+    it "should be global even when datacenter uses renewable energy" do
+      SWD.new(AVERAGE_WEBSITE_IN_BYTES, true).device_carbon_intensity.should(eq SWD::GLOBAL_GRID_INTENSITY)
+    end
+
+    it "can be initialized with a country iso code" do
+      SWD.new(AVERAGE_WEBSITE_IN_BYTES, false, nil, "AR").device_carbon_intensity.should(eq SWD::IntensityData.by_iso("AR"))
+    end
+  end
+
+  describe "#network_carbon_intensity" do
+    it "should be a global value by default" do
+      swd.network_carbon_intensity.should(eq SWD::GLOBAL_GRID_INTENSITY)
+    end
+
+    it "should be global even when datacenter uses renewable energy" do
+      SWD.new(AVERAGE_WEBSITE_IN_BYTES, true).network_carbon_intensity.should(eq SWD::GLOBAL_GRID_INTENSITY)
+    end
+  end
+
+  describe "#global_emissions" do
+    it "should be a global value by default" do
+      swd.global_emissions.should(eq SWD::GLOBAL_GRID_INTENSITY)
+    end
+
+    it "should be global even when datacenter uses renewable energy" do
+      SWD.new(AVERAGE_WEBSITE_IN_BYTES, true).global_emissions.should(eq SWD::GLOBAL_GRID_INTENSITY)
+    end
+  end
+
+  describe "#consumer_device_co2" do
+    it "should extract the consumer device co2 emissions" do
+      swd.consumer_device_co2.should(eq 0.6406335995061729)
+    end
+  end
+
+  describe "#production_co2" do
+    it "should extract the production co2 emissions" do
+      swd.production_co2.should(eq 0.23407766135802469)
+    end
+  end
+
+  describe "#network_co2" do
+    it "should extract the network co2 emissions" do
+      swd.network_co2.should(eq 0.17247827679012348)
+    end
+  end
+
+  describe "#datacenter_co2" do
+    it "should extract the datacenter co2 emissions" do
+      swd.datacenter_co2.should(eq 0.1847981537037037)
+    end
+  end
+
+  describe "#total_co2" do
+    it "should extract the total co2 emissions" do
+      swd.total_co2.should(eq 1.2319876913580248)
+    end
+  end
+end

src/access_log.cr

@@ -1,3 +1,4 @@
+require "log"
 require "file_utils"
 require "json"
 require "socket"
@@ -5,10 +6,14 @@ require "sqlite3"
 require "pg"
 require "option_parser"
 require "uuid"
+require "system"
 require "./models/access_log"
 require "./models/crawler"
+require "./swd"
 
-VERSION = "0.4.0"
+VERSION = "0.5.5"
+
+Log.setup_from_env
 
 # Default socket location
 socket = "/tmp/access_log.socket"
@@ -20,9 +25,17 @@ crawler = false
 crawlers = [] of Crawler
 # Fields in database
 # TODO: Obtain them from AccessLog
-fields = ["id", "remote_user", "host", "msec", "server_protocol", "request_method", "request_completion", "uri", "request_uri", "query_string", "status", "sent_http_content_type", "sent_http_content_encoding", "sent_http_etag", "sent_http_last_modified", "http_accept", "http_accept_encoding", "http_accept_language", "http_pragma", "http_cache_control", "http_if_none_match", "http_dnt", "http_user_agent", "http_origin", "http_referer", "request_time", "bytes_sent", "body_bytes_sent", "request_length", "http_connection", "pipe", "connection_requests", "geoip2_data_country_name", "geoip2_data_city_name", "ssl_server_name", "ssl_protocol", "ssl_early_data", "ssl_session_reused", "ssl_curves", "ssl_ciphers", "ssl_cipher", "sent_http_x_xss_protection", "sent_http_x_frame_options", "sent_http_x_content_type_options", "sent_http_strict_transport_security", "nginx_version", "pid", "crawler"]
+fields = %w[id remote_user host msec server_protocol request_method request_completion uri request_uri query_string status sent_http_content_type sent_http_content_encoding sent_http_etag sent_http_last_modified http_accept http_accept_encoding http_accept_language http_pragma http_cache_control http_if_none_match http_dnt http_user_agent http_origin http_referer request_time bytes_sent body_bytes_sent request_length http_connection pipe connection_requests geoip2_data_country_name geoip2_data_city_name ssl_server_name ssl_protocol ssl_early_data ssl_session_reused ssl_curves ssl_ciphers ssl_cipher sent_http_x_xss_protection sent_http_x_frame_options sent_http_x_content_type_options sent_http_strict_transport_security nginx_version pid crawler datacenter_co2 network_co2 consumer_device_co2 production_co2 total_co2 node]
 # Params for the query
 params = [] of String
+# SWD
+swd = false
+# Datacenter
+datacenter : String? = nil
+renewable = false
+device_country = false
+intensity = :marginal
+node = false
 
 # Parse CLI flags
 OptionParser.parse do |p|
@@ -51,10 +64,34 @@ OptionParser.parse do |p|
   p.on "-s /tmp/access_log.socket", "--socket /tmp/access_log.socket", "Listening socket" do |s|
     socket = s
   end
+
+  p.on "-S", "--swd", "Enable Sustainable Web Design CO2 emission tracking" do
+    swd = true
+  end
+
+  p.on "-D ISO2", "--datacenter ISO2", "Datacenter ISO 2-letter country code" do |d|
+    datacenter = d
+  end
+
+  p.on "-R", "--renewable", "Datacenter uses renewable energy" do
+    renewable = true
+  end
+
+  p.on "-C", "--device-country", "Take country of visit for SWD" do
+    device_country = true
+  end
+
+  p.on "-A", "--average-intensity", "Use average intensity figures rather than marginal" do
+    intensity = :average
+  end
+
+  p.on "-n", "--node", "Send node hostname" do
+    node = true
+  end
 end
 
 # Parameterize values according to database URI
-if database.starts_with? "postgresql://"
+if database.starts_with?("postgresql://") || database.starts_with?("postgres://")
   params = fields.map_with_index do |_, i|
     "$#{i + 1}"
   end
@@ -93,10 +130,13 @@ db = DB.open database do |db|
   while true
     begin
       msg, _ = server.receive(1024 * 64) # 64K
-      _, _, _, _, json = msg.split(" ", 5)
+      _, json = msg.split("{", 2)
+      json = "{#{json}"
 
       # Parse input
       access_log = AccessLog.from_json(json || "{}")
+      d = device_country ? access_log.geoip2_data_country_iso_code : nil
+      s = SWD.new(access_log.bytes_sent, renewable, datacenter, d, intensity) if swd
 
       # Execute query, detect crawler if enabled.
       # TODO: Does Crystal support splats?  It does but we need to cast
@@ -149,10 +189,17 @@ db = DB.open database do |db|
               access_log.sent_http_strict_transport_security,
               access_log.nginx_version,
               access_log.pid,
-              (crawler ? !!(crawler_re =~ access_log.http_user_agent) : false)
+              (crawler ? !!(crawler_re =~ access_log.http_user_agent) : false),
+              (swd ? s.try(&.datacenter_co2) : nil),
+              (swd ? s.try(&.network_co2) : nil),
+              (swd ? s.try(&.consumer_device_co2) : nil),
+              (swd ? s.try(&.production_co2) : nil),
+              (swd ? s.try(&.total_co2) : nil),
+              (node ? System.hostname : nil)
 
     # Ignore parsing errors
-    rescue JSON::ParseException
+    rescue e : JSON::ParseException
+      Log.warn &.emit("Parse exception", error: e.message)
     rescue IO::Error
       server.close
       FileUtils.rm(socket) if File.exists? socket

src/average_intensities_by_country.cr

@@ -0,0 +1,14 @@
+require "json"
+
+average_intensities = Hash(String, Float32).new
+
+JSON.parse(ARGF.gets_to_end).as_h.each do |country, data|
+  if data["country_code_iso_2"].as_s.empty?
+    STDERR.puts "not ok - Skipping #{country} because of missing ISO code"
+    next
+  end
+
+  average_intensities[data["country_code_iso_2"].as_s] = data["emissions_intensity_gco2_per_kwh"].as_s.to_f32
+end
+
+puts average_intensities.to_json

src/marginal_intensities_by_country.cr

@@ -0,0 +1,14 @@
+require "json"
+
+marginal_intensities = Hash(String, Float32).new
+
+JSON.parse(ARGF.gets_to_end).as_h.each do |country, data|
+  if data["country_code_iso_2"].as_s.empty?
+    STDERR.puts "not ok - Skipping #{country} because of missing ISO code"
+    next
+  end
+
+  marginal_intensities[data["country_code_iso_2"].as_s] = data["Operating Margin Grid Emission"].as_s.to_f32
+end
+
+puts marginal_intensities.to_json

src/models/access_log.cr

@@ -43,6 +43,7 @@ class AccessLog
   property pipe : String
   property connection_requests : Int32
   property geoip2_data_country_name : String
+  property geoip2_data_country_iso_code : String?
   property geoip2_data_city_name : String
   property ssl_server_name : String
   property ssl_protocol : String

src/swd.cr

@@ -0,0 +1,126 @@
+# This is an adaptation of the Sustainable Web Design algorithm to
+# calculate CO2 emissions per visit.
+#
+# In this case if we know the country of origin of the visit we can
+# adapt the device emissions to the country, same as the datacenter
+# variable on the original proposal.
+#
+# Also the CO2 per visit is adapted to the actual cache, since we know
+# the status code.  304 visits are cached so the energy usage is much
+# smaller and we don't need to make assumptions.
+#
+# @see https://sustainablewebdesign.org/calculating-digital-emissions/
+
+require "./swd/intensity_data"
+
+class SWD
+  getter bytes : Int8 | Int16 | Int32 | Int64 | UInt8 | UInt16 | UInt32 | UInt64
+  getter renewable : Bool
+  getter datacenter_iso_code : String?
+  getter consumer_device_iso_code : String?
+  getter intensity_calculator : Symbol
+
+  GIGABYTE = 1000.0 * 1000.0 * 1000.0
+  KWH_PER_GB = 0.81
+  END_USER_DEVICE_ENERGY = 0.52
+  NETWORK_ENERGY = 0.14
+  DATACENTER_ENERGY = 0.15
+  PRODUCTION_ENERGY = 0.19
+  GLOBAL_GRID_INTENSITY = 442.0
+  RENEWABLES_GRID_INTENSITY = 50.0
+
+  @transfered_bytes_to_gb : Float32? | Float64? = nil
+  @energy_usage : Float32? | Float64? = nil
+  @consumer_device_energy : Float32? | Float64? = nil
+  @network_energy : Float32? | Float64? = nil
+  @production_energy : Float32? | Float64? = nil
+  @datacenter_energy : Float32? | Float64? = nil
+  @total_energy : Float32? | Float64? = nil
+  @consumer_device_co2 : Float32? | Float64? = nil
+  @network_co2 : Float32? | Float64? = nil
+  @production_co2 : Float32? | Float64? = nil
+  @datacenter_co2 : Float32? | Float64? = nil
+  @total_co2 : Float32? | Float64? = nil
+
+  def initialize(@bytes, @renewable = false, @datacenter_iso_code = nil, @consumer_device_iso_code = nil, @intensity_calculator = :marginal); end
+
+  def transfered_bytes_to_gb
+    @transfered_bytes_to_gb ||= @bytes / GIGABYTE
+  end
+
+  def energy_usage
+    @energy_usage ||= transfered_bytes_to_gb / KWH_PER_GB
+  end
+
+  def consumer_device_energy
+    @consumer_device_energy ||= energy_usage * END_USER_DEVICE_ENERGY
+  end
+
+  def network_energy
+    @network_energy ||= energy_usage * NETWORK_ENERGY
+  end
+
+  def production_energy
+    @production_energy ||= energy_usage * PRODUCTION_ENERGY
+  end
+
+  def total_energy
+    @total_energy ||= production_energy + network_energy + datacenter_energy + consumer_device_energy
+  end
+
+  def datacenter_energy
+    @datacenter_energy ||= energy_usage * DATACENTER_ENERGY
+  end
+
+  # Returns the carbon intensity depending on renewable status and
+  # location, by default uses global grid intensity.
+  def datacenter_carbon_intensity : Float
+    renewable ? RENEWABLES_GRID_INTENSITY : (localized_datacenter_intensity || GLOBAL_GRID_INTENSITY)
+  end
+
+  # Returns the carbon intensity of user devices depending on location,
+  # by default uses global grid intensity
+  def device_carbon_intensity : Float
+    localized_consumer_device_intensity || GLOBAL_GRID_INTENSITY
+  end
+
+  def network_carbon_intensity : Float
+    GLOBAL_GRID_INTENSITY
+  end
+
+  def global_emissions : Float
+    GLOBAL_GRID_INTENSITY
+  end
+
+  def datacenter_co2
+    @datacenter_co2 ||= datacenter_carbon_intensity * datacenter_energy
+  end
+
+  def consumer_device_co2
+    @consumer_device_co2 ||= device_carbon_intensity * consumer_device_energy
+  end
+
+  def network_co2
+    @network_co2 ||= network_carbon_intensity * network_energy
+  end
+
+  def production_co2
+    @production_co2 ||= production_energy * global_emissions
+  end
+
+  def total_co2
+    @total_co2 ||= production_co2 + network_co2 + datacenter_co2 + consumer_device_co2
+  end
+
+  def localized_datacenter_intensity : Float64?
+    return unless datacenter_iso_code
+
+    SWD::IntensityData.by_iso(datacenter_iso_code || "", intensity_calculator)
+  end
+
+  def localized_consumer_device_intensity : Float64?
+    return unless consumer_device_iso_code
+
+    SWD::IntensityData.by_iso(consumer_device_iso_code || "", intensity_calculator)
+  end
+end

src/swd/intensity_data.cr

@@ -0,0 +1,28 @@
+class SWD
+  module IntensityData
+    # @see {co2.js/data/output/average-intensities-2021.json}
+    AVERAGE_INTENSITY_BY_ISO_CODE = {"AR": 365.292, "AM": 206.522, "AU": 526.876, "AT": 145.083, "AZ": 536.585, "BD": 559.606, "BY": 472.727, "BE": 156.063, "BO": 311.475, "BA": 470.982, "BR": 144.677, "BG": 364.136, "BI": 275.862, "CA": 123.859, "CL": 395.565, "CN": 549.288, "CR": 30.903, "HR": 212.161, "CY": 601.19, "CZ": 401.272, "DK": 240.419, "EC": 132.964, "EG": 470.879, "SV": 180.87, "EE": 488.529, "FI": 152.651, "FR": 67.781, "GE": 105.685, "DE": 363.982, "GR": 363.388, "HU": 236.271, "IN": 632.656, "IE": 361.274, "IT": 340.937, "JP": 460.647, "KZ": 656.097, "KE": 104.0, "LV": 226.351, "LT": 247.475, "LU": 183.824, "MT": 452.055, "MX": 391.582, "MD": 642.512, "MN": 725.26, "ME": 335.958, "NL": 386.189, "MK": 444.191, "NO": 26.131, "PK": 363.065, "PE": 241.492, "PH": 579.689, "PL": 657.138, "PT": 222.632, "RO": 255.718, "RU": 355.431, "SA": 568.967, "SN": 540.098, "RS": 549.083, "SG": 488.21, "SK": 173.854, "SI": 241.956, "ZA": 706.991, "KR": 442.389, "ES": 193.737, "SE": 43.9, "CH": 58.952, "TW": 565.629, "TJ": 72.823, "TH": 503.034, "TN": 470.848, "TR": 432.293, "UA": 240.28, "GB": 268.255, "US": 378.625, "VN": 491.192}
+
+    # @see {co2.js/data/output/marginal-intensities-2021.json}
+    MARGINAL_INTENSITY_BY_ISO_CODE = {"AF": 414.0, "AL": 0.0, "DZ": 528.0, "AS": 753.0, "AD": 188.0, "AO": 1476.0, "AI": 753.0, "AG": 753.0, "AR": 478.0, "AM": 390.0, "AW": 753.0, "AU": 808.0, "AT": 242.0, "AZ": 534.0, "BS": 753.0, "BH": 726.0, "BD": 528.0, "BB": 749.0, "BY": 400.0, "BE": 252.0, "BZ": 403.0, "BJ": 745.0, "BM": 753.0, "BT": 0.0, "BO": 604.0, "BQ": 753.0, "BA": 1197.0, "BW": 1486.0, "BR": 284.0, "VG": 753.0, "BN": 681.0, "BG": 911.0, "BF": 753.0, "BI": 414.0, "KH": 1046.0, "CM": 659.0, "CA": 372.0, "KY": 753.0, "CV": 753.0, "CY": 751.0, "CF": 188.0, "TD": 753.0, "CL": 657.0, "CN": 899.0, "CO": 410.0, "KM": 753.0, "CD": 0.0, "CG": 659.0, "CK": 753.0, "CR": 108.0, "CI": 466.0, "HR": 294.0, "CU": 559.0, "CW": 876.0, "CZ": 902.0, "DK": 362.0, "DJ": 753.0, "DM": 753.0, "DO": 601.0, "EC": 560.0, "EG": 554.0, "SV": 547.0, "GQ": 632.0, "ER": 915.0, "EE": 1057.0, "SZ": 0.0, "ET": 0.0, "FK": 753.0, "FO": 753.0, "FJ": 640.0, "FI": 267.0, "FR": 158.0, "GF": 423.0, "PF": 753.0, "GA": 946.0, "GM": 753.0, "GE": 289.0, "DE": 650.0, "GH": 495.0, "GI": 779.0, "GR": 507.0, "GL": 264.0, "GD": 753.0, "GP": 753.0, "GU": 753.0, "GT": 798.0, "GN": 753.0, "GW": 753.0, "GY": 847.0, "HT": 1048.0, "HN": 662.0, "HU": 296.0, "IS": 0.0, "IN": 951.0, "ID": 783.0, "IR": 592.0, "IQ": 1080.0, "IE": 380.0, "IM": 436.0, "IL": 394.0, "IT": 414.0, "JM": 711.0, "JP": 471.0, "JO": 529.0, "KZ": 797.0, "KE": 574.0, "KI": 753.0, "KP": 754.0, "KR": 555.0, "XK": 1145.0, "KW": 675.0, "KG": 217.0, "LA": 1069.0, "LV": 240.0, "LB": 794.0, "LS": 0.0, "LR": 677.0, "LY": 668.0, "LI": 151.0, "LT": 211.0, "LU": 220.0, "MG": 876.0, "MW": 489.0, "MY": 551.0, "MV": 753.0, "ML": 1076.0, "MT": 520.0, "MH": 753.0, "MQ": 753.0, "MR": 753.0, "MU": 700.0, "YT": 753.0, "MX": 531.0, "FM": 753.0, "MD": 541.0, "MC": 158.0, "MN": 1366.0, "ME": 899.0, "MS": 753.0, "MA": 729.0, "MZ": 234.0, "MM": 719.0, "NA": 355.0, "NR": 753.0, "NP": 0.0, "NL": 326.0, "NC": 779.0, "NZ": 246.0, "NI": 675.0, "NE": 772.0, "NG": 526.0, "NU": 753.0, "MK": 851.0, "MP": 753.0, "NO": 47.0, "OM": 479.0, "PK": 592.0, "PW": 753.0, "PS": 719.0, "PA": 477.0, "PG": 597.0, "PY": 0.0, "PE": 473.0, "PH": 672.0, "PL": 828.0, "PT": 389.0, "PR": 596.0, "QA": 503.0, "RE": 772.0, "RO": 489.0, "RU": 476.0, "RW": 712.0, "SH": 753.0, "KN": 753.0, "LC": 753.0, "MF": 753.0, "PM": 753.0, "VC": 753.0, "WS": 753.0, "SM": 414.0, "ST": 753.0, "SA": 592.0, "SN": 870.0, "RS": 1086.0, "SC": 753.0, "SL": 489.0, "SG": 379.0, "SX": 753.0, "SK": 332.0, "SI": 620.0, "SB": 753.0, "SO": 753.0, "ZA": 1070.0, "SS": 890.0, "ES": 402.0, "LK": 731.0, "SD": 736.0, "SR": 1029.0, "SE": 68.0, "CH": 48.0, "SY": 713.0, "TW": 484.0, "TJ": 255.0, "TZ": 531.0, "TH": 450.0, "TL": 753.0, "TG": 859.0, "TO": 753.0, "TT": 559.0, "TN": 468.0, "TR": 376.0, "TM": 927.0, "TC": 753.0, "TV": 753.0, "UG": 279.0, "UA": 768.0, "AE": 556.0, "GB": 380.0, "US": 416.0, "UY": 174.0, "UZ": 612.0, "VU": 753.0, "VE": 711.0, "VN": 560.0, "VI": 650.0, "YE": 807.0, "ZM": 416.0, "ZW": 1575.0}
+
+    # Return the marginal intensity for a country when provided an ISO
+    # code
+    def self.marginal_by_iso(iso : String) : Float64?
+      MARGINAL_INTENSITY_BY_ISO_CODE[iso] if MARGINAL_INTENSITY_BY_ISO_CODE.has_key? iso
+    end
+
+    # Return the average intensity for a country when provided an ISO
+    # code
+    def self.average_by_iso(iso : String) : Float64?
+      AVERAGE_INTENSITY_BY_ISO_CODE[iso] if AVERAGE_INTENSITY_BY_ISO_CODE.has_key? iso
+    end
+
+    def self.by_iso(iso : String, type : Symbol = :marginal) : Float64?
+      case type
+      when :average then average_by_iso(iso)
+      else marginal_by_iso(iso)
+      end
+    end
+  end
+end