fix: tests

https://github.com/thegreenwebfoundation/co2.js/commit/b0a65fe073c8a4d7fc2adbafb1844e4cae3ee6c2

README.md

@@ -43,6 +43,22 @@ most countries.
 
 `access_log` uses marginal data by default.
 
+### Update the data
+
+1. Go to the `data/output` directory on [co2.js
+   repository](https://github.com/thegreenwebfoundation/co2.js/tree/v0.16.2/data/output)
+   for the latest version released (in this example 0.16.2)
+
+2. Download the `average-intensities.json` file.
+
+3. Run `src/average_intensities_by_country.cr` with this file as stdin.
+
+```
+crystal run src/average_intensities_by_country.cr < average-intensities.json >> src/swd/average_intensity.cr
+```
+
+4. Modify `src/swd/average_intensity.cr` to fix the data.
+
 ## Create database
 
 ```bash
@@ -179,6 +195,18 @@ Then run the program with the required flags enabled:
 access_log --swd --device-country
 ```
 
+## ASN database
+
+If you want to keep track of ASN for each visitor, for instance for
+grouping possible attacks or IA crawls, create a database based on
+<https://iptoasn.com/>:
+
+```bash
+./contrib/asn_database.sh
+```
+
+And start the server with the `--asn-database=` flag.
+
 ## Crawler user agents
 
 Download the [crawler user agents

contrib/asn_database.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -e
+
+db="${1:-$PWD/asn.sqlite3}"
+sqlite3 "$db" "CREATE TABLE IF NOT EXISTS asn (range_start INTEGER, range_end INTEGER, id INTEGER, country STRING, description TEXT);"
+sqlite3 "$db" "DELETE from asn;"
+wget https://iptoasn.com/data/ip2asn-v4-u32.tsv.gz -O - | gunzip | sqlite3 -tabs "$db" ".import '|cat -' asn"
+
+echo "ASN database created/updated at $db"

contrib/create.sql

@@ -53,7 +53,9 @@ CREATE TABLE IF NOT EXISTS "access_logs" (
  "network_co2" float DEFAULT NULL,
  "consumer_device_co2" float DEFAULT NULL,
  "production_co2" float DEFAULT NULL,
- "total_co2" float DEFAULT NULL
+ "total_co2" float DEFAULT NULL,
+ "node" varchar DEFAULT NULL,
+ "asn" integer DEFAULT NULL
 );
 
 CREATE INDEX IF NOT EXISTS "index_access_logs_on_host" ON "access_logs" ("host");

shard.yml

@@ -1,5 +1,5 @@
 name: "access_log"
-version: "0.5.6"
+version: "0.6.0"
 authors:
 - "f <f@sutty.nl>"
 targets:

spec/swd_spec.cr

@@ -111,31 +111,31 @@ describe SWD do
 
   describe "#consumer_device_co2" do
     it "should extract the consumer device co2 emissions" do
-      swd.consumer_device_co2.should(eq 0.6343432152938272)
+      swd.consumer_device_co2.should(eq 0.6854924454444445)
     end
   end
 
   describe "#production_co2" do
     it "should extract the production co2 emissions" do
-      swd.production_co2.should(eq 0.2317792517419753)
+      swd.production_co2.should(eq 0.2504683935277778)
     end
   end
 
   describe "#network_co2" do
     it "should extract the network co2 emissions" do
-      swd.network_co2.should(eq 0.1707847118098766)
+      swd.network_co2.should(eq 0.1845556583888889)
     end
   end
 
   describe "#datacenter_co2" do
     it "should extract the datacenter co2 emissions" do
-      swd.datacenter_co2.should(eq 0.1829836197962963)
+      swd.datacenter_co2.should(eq 0.19773820541666665)
     end
   end
 
   describe "#total_co2" do
     it "should extract the total co2 emissions" do
-      swd.total_co2.should(eq 1.2198907986419754)
+      swd.total_co2.should(eq 1.3182547027777778)
     end
   end
 end

src/access_log.cr

@@ -10,8 +10,9 @@ require "system"
 require "./models/access_log"
 require "./models/crawler"
 require "./swd"
+require "./asn"
 
-VERSION = "0.5.7"
+VERSION = "0.6.0"
 
 Log.setup_from_env
 
@@ -19,13 +20,15 @@ Log.setup_from_env
 socket = "/tmp/access_log.socket"
 # The default database URI
 database = "sqlite3://./development.sqlite3"
+# ASN database
+asn_database = ""
 # Detect web crawlers
 crawler = false
 # Parse the crawlers repository
 crawlers = [] of Crawler
 # Fields in database
 # TODO: Obtain them from AccessLog
-fields = %w[id remote_user host msec server_protocol request_method request_completion uri request_uri query_string status sent_http_content_type sent_http_content_encoding sent_http_etag sent_http_last_modified http_accept http_accept_encoding http_accept_language http_pragma http_cache_control http_if_none_match http_dnt http_user_agent http_origin http_referer request_time bytes_sent body_bytes_sent request_length http_connection pipe connection_requests geoip2_data_country_name geoip2_data_city_name ssl_server_name ssl_protocol ssl_early_data ssl_session_reused ssl_curves ssl_ciphers ssl_cipher sent_http_x_xss_protection sent_http_x_frame_options sent_http_x_content_type_options sent_http_strict_transport_security nginx_version pid crawler datacenter_co2 network_co2 consumer_device_co2 production_co2 total_co2 node]
+fields = %w[id remote_user host msec server_protocol request_method request_completion uri request_uri query_string status sent_http_content_type sent_http_content_encoding sent_http_etag sent_http_last_modified http_accept http_accept_encoding http_accept_language http_pragma http_cache_control http_if_none_match http_dnt http_user_agent http_origin http_referer request_time bytes_sent body_bytes_sent request_length http_connection pipe connection_requests geoip2_data_country_name geoip2_data_city_name ssl_server_name ssl_protocol ssl_early_data ssl_session_reused ssl_curves ssl_ciphers ssl_cipher sent_http_x_xss_protection sent_http_x_frame_options sent_http_x_content_type_options sent_http_strict_transport_security nginx_version pid crawler datacenter_co2 network_co2 consumer_device_co2 production_co2 total_co2 node asn]
 # Params for the query
 params = [] of String
 # SWD
@@ -56,6 +59,10 @@ OptionParser.parse do |p|
     database = d
   end
 
+  p.on "-I DATABASE", "--asn-database=DATABASE", "ASN Database URI" do |d|
+    asn_database = d
+  end
+
   p.on "-c crawler-user-agents.json", "--crawlers crawler-user-agents.json", "Crawlers repository" do |c|
     crawler = true
     crawlers = Array(Crawler).from_json File.read(c)
@@ -110,26 +117,41 @@ if crawler
   crawler_re = Regex.union(crawlers.map { |c| c.pattern })
 end
 
+def remove_socket!(socket)
+  FileUtils.rm(socket) if File.exists? socket
+end
+
+remove_socket!(socket)
+
+asn = ASN.new(asn_database) unless asn_database.empty?
 server = Socket.unix(Socket::Type::DGRAM)
 server.bind Socket::UNIXAddress.new(socket)
 
+def close_server!(server, asn)
+  server.try(&.close)
+  asn.try(&.close)
+end
+
 Signal::INT.trap do
-  server.close
-  FileUtils.rm(socket) if File.exists? socket
+  remove_socket!(socket)
+  close_server!(server, asn)
   exit
 end
 
 Signal::KILL.trap do
-  server.close
-  FileUtils.rm(socket) if File.exists? socket
+  remove_socket!(socket)
+  close_server!(server, asn)
   exit
 end
 
+MAX_SIZE = 1024 * 64 # 64K
+
 # Open the database and wait for JSONL input.
-db = DB.open database do |db|
+DB.open database do |db|
   while true
     begin
-      msg, _ = server.receive(1024 * 64) # 64K
+      msg, _ = server.receive(MAX_SIZE)
+      # TODO: is this the best method?
       _, json = msg.split("{", 2)
       json = "{#{json}"
 
@@ -137,6 +159,8 @@ db = DB.open database do |db|
       access_log = AccessLog.from_json(json || "{}")
       d = device_country ? access_log.geoip2_data_country_iso_code : nil
       s = SWD.new(access_log.bytes_sent, renewable, datacenter, d, intensity) if swd
+      i = access_log.remote_addr
+      a = asn.try(&.query(i)) if i
 
       # Execute query, detect crawler if enabled.
       # TODO: Does Crystal support splats?  It does but we need to cast
@@ -195,14 +219,15 @@ db = DB.open database do |db|
               (swd ? s.try(&.consumer_device_co2) : nil),
               (swd ? s.try(&.production_co2) : nil),
               (swd ? s.try(&.total_co2) : nil),
-              (node ? System.hostname : nil)
+              (node ? System.hostname : nil),
+              a.try(&.id)
 
     # Ignore parsing errors
     rescue e : JSON::ParseException
       Log.warn &.emit("Parse exception", error: e.message)
     rescue IO::Error
-      server.close
-      FileUtils.rm(socket) if File.exists? socket
+      remove_socket!(socket)
+      close_server!(server, asn)
       exit
     end
   end

src/asn.cr

@@ -0,0 +1,41 @@
+require "db"
+require "sqlite3"
+
+class AsnResult
+  include DB::Serializable
+
+  property id : Int64?
+  property country : String?
+  property description : String?
+end
+
+class ASN
+  QUERY = "select id, country, description from asn where ? between range_start and range_end limit 1;"
+
+  property db : DB::Database
+
+  def initialize(database : String)
+    @db = DB.open(database)
+    @cache = Hash(UInt32, AsnResult).new
+  end
+
+  # Binary representation of an IPv4 address
+  def to_binary(address : String) : UInt32
+    (address.split(".").map(&.to_u32).reduce(0.to_u32) { |t, v| (t << 8) + v })
+  end
+
+  # Lookups an address and returns the ASN for it
+  def query(address : String)
+    query(to_binary(address))
+  end
+
+  def query(binary_address : UInt32) : AsnResult?
+    @cache[binary_address] ||= @db.query_one(QUERY, binary_address.to_i64, &.read(AsnResult))
+  rescue DB::NoResultsError
+    nil
+  end
+
+  def close
+    @db.close
+  end
+end

src/average_intensities_by_country.cr

@@ -23,7 +23,12 @@ JSON.parse(ARGF.gets_to_end).as_h.each do |country, data|
 
   STDERR.puts "ok - #{country}"
 
-  average_intensities[countries[data["country_code"].as_s]] = data["emissions_intensity_gco2_per_kwh"].as_f
+  average_intensities[countries[data["country_code"].as_s]] =
+    begin
+      data["emissions_intensity_gco2_per_kwh"].as_f
+    rescue TypeCastError
+      data["emissions_intensity_gco2_per_kwh"].as_i.to_f
+    end
 end
 
 puts average_intensities.to_json

src/models/access_log.cr

@@ -58,4 +58,5 @@ class AccessLog
   property sent_http_strict_transport_security : String
   property nginx_version : String
   property pid : String
+  property remote_addr : String?
 end

src/swd.cr

@@ -27,7 +27,7 @@ class SWD
   NETWORK_ENERGY = 0.14
   DATACENTER_ENERGY = 0.15
   PRODUCTION_ENERGY = 0.19
-  GLOBAL_GRID_INTENSITY = 437.66
+  GLOBAL_GRID_INTENSITY = 472.95
   RENEWABLES_GRID_INTENSITY = 50.0
 
   @transfered_bytes_to_gb : Float32? | Float64? = nil

src/swd/intensity_data.cr

@@ -2,215 +2,215 @@ class SWD
   module IntensityData
     # @see {co2.js/data/output/average-intensities.json}
     AVERAGE_INTENSITY_BY_ISO_CODE = {
-      "AE": 561.14,
-      "AF": 120.48,
-      "AG": 657.14,
-      "AL": 23.44,
-      "AM": 280.06,
-      "AO": 212.42,
-      "AR": 399.5,
-      "AS": 687.5,
-      "AT": 141.79,
-      "AU": 570.35,
-      "AW": 591.4,
-      "AZ": 719.39,
-      "BA": 670.09,
-      "BB": 644.86,
-      "BD": 719.78,
-      "BE": 139.79,
-      "BF": 611.43,
-      "BG": 476.9,
-      "BH": 956.55,
-      "BI": 250.0,
-      "BJ": 666.67,
-      "BN": 961.54,
-      "BO": 497.78,
-      "BR": 114.48,
-      "BS": 698.11,
-      "BT": 24.44,
-      "BW": 867.58,
-      "BY": 465.34,
-      "BZ": 484.38,
-      "CA": 163.01,
-      "CD": 25.36,
+      "AE": 467.51,
+      "AF": 123.71,
+      "AG": 611.11,
+      "AL": 24.42,
+      "AM": 243.52,
+      "AO": 167.22,
+      "AR": 344.83,
+      "AS": 647.06,
+      "AT": 102.62,
+      "AU": 553.76,
+      "AW": 550.0,
+      "AZ": 632.89,
+      "BA": 638.05,
+      "BB": 600.0,
+      "BD": 694.63,
+      "BE": 117.58,
+      "BF": 554.91,
+      "BG": 264.21,
+      "BH": 902.41,
+      "BI": 230.77,
+      "BJ": 590.0,
+      "BN": 892.67,
+      "BO": 468.02,
+      "BR": 106.06,
+      "BS": 653.66,
+      "BT": 24.19,
+      "BW": 849.42,
+      "BY": 323.63,
+      "BZ": 155.56,
+      "CA": 184.99,
+      "CD": 27.04,
       "CF": 0.0,
-      "CG": 723.88,
-      "CH": 36.14,
-      "CI": 453.55,
-      "CK": 400.0,
-      "CL": 353.52,
-      "CM": 278.26,
-      "CN": 585.82,
-      "CO": 198.53,
-      "CR": 37.21,
-      "CU": 677.47,
-      "CV": 600.0,
-      "CY": 555.13,
-      "CZ": 488.3,
-      "DE": 420.06,
-      "DJ": 666.67,
-      "DK": 201.78,
-      "DM": 529.41,
-      "DO": 592.49,
-      "DZ": 672.64,
-      "EC": 192.39,
-      "EE": 489.89,
-      "EG": 612.16,
-      "ER": 688.89,
-      "ES": 217.77,
-      "ET": 25.19,
-      "FI": 130.42,
-      "FJ": 289.47,
-      "FK": 500.0,
-      "FO": 428.57,
-      "FR": 78.81,
-      "GA": 397.38,
-      "GB": 255.85,
-      "GD": 714.29,
-      "GE": 176.97,
-      "GF": 254.72,
-      "GH": 498.81,
-      "GL": 133.33,
-      "GM": 700.0,
-      "GN": 208.63,
-      "GP": 647.06,
-      "GQ": 492.96,
-      "GR": 378.18,
-      "GT": 331.72,
-      "GU": 670.33,
-      "GW": 750.0,
-      "GY": 642.28,
-      "HK": 733.05,
-      "HN": 378.97,
-      "HR": 242.28,
-      "HT": 606.06,
-      "HU": 228.75,
-      "ID": 675.93,
-      "IE": 333.23,
-      "IL": 641.17,
-      "IN": 705.13,
-      "IQ": 708.1,
-      "IR": 665.15,
-      "IS": 28.56,
-      "IT": 378.45,
-      "JM": 588.51,
-      "JO": 572.74,
-      "JP": 512.81,
-      "KE": 101.13,
-      "KG": 124.48,
-      "KH": 436.13,
-      "KI": 666.67,
-      "KM": 714.29,
-      "KN": 681.82,
-      "KP": 124.57,
-      "KR": 470.4,
-      "KW": 691.31,
-      "KY": 684.93,
-      "KZ": 830.41,
-      "LA": 284.21,
-      "LB": 663.1,
-      "LC": 685.71,
-      "LK": 543.12,
-      "LR": 304.35,
-      "LS": 20.0,
-      "LT": 177.18,
-      "LU": 144.07,
-      "LV": 136.27,
-      "LY": 861.69,
-      "MA": 705.67,
-      "MD": 871.66,
-      "ME": 483.38,
-      "MG": 497.61,
-      "MK": 639.86,
-      "ML": 463.13,
-      "MM": 575.77,
-      "MN": 852.82,
-      "MO": 627.12,
-      "MQ": 698.63,
-      "MR": 526.6,
+      "CG": 713.73,
+      "CH": 36.72,
+      "CI": 393.53,
+      "CK": 250.0,
+      "CL": 265.52,
+      "CM": 285.71,
+      "CN": 557.5,
+      "CO": 285.8,
+      "CR": 63.01,
+      "CU": 638.98,
+      "CV": 480.0,
+      "CY": 512.24,
+      "CZ": 413.86,
+      "DE": 342.06,
+      "DJ": 450.0,
+      "DK": 143.3,
+      "DM": 600.0,
+      "DO": 565.97,
+      "DZ": 633.65,
+      "EC": 209.7,
+      "EE": 341.02,
+      "EG": 574.5,
+      "ER": 590.91,
+      "ES": 146.15,
+      "ET": 23.55,
+      "FI": 72.25,
+      "FJ": 278.26,
+      "FK": 1000.0,
+      "FO": 354.17,
+      "FR": 44.18,
+      "GA": 429.47,
+      "GB": 215.79,
+      "GD": 666.67,
+      "GE": 143.06,
+      "GF": 204.08,
+      "GH": 452.86,
+      "GL": 111.11,
+      "GM": 666.67,
+      "GN": 182.72,
+      "GP": 493.9,
+      "GQ": 605.1,
+      "GR": 319.76,
+      "GT": 272.66,
+      "GU": 611.11,
+      "GW": 625.0,
+      "GY": 634.33,
+      "HK": 681.99,
+      "HN": 289.5,
+      "HR": 174.48,
+      "HT": 534.65,
+      "HU": 182.82,
+      "ID": 682.43,
+      "IE": 279.79,
+      "IL": 567.26,
+      "IN": 708.96,
+      "IQ": 689.4,
+      "IR": 648.68,
+      "IS": 28.33,
+      "IT": 287.75,
+      "JM": 561.25,
+      "JO": 539.21,
+      "JP": 483.73,
+      "KE": 84.83,
+      "KG": 152.65,
+      "KH": 497.46,
+      "KI": 500.0,
+      "KM": 642.86,
+      "KN": 636.36,
+      "KP": 344.26,
+      "KR": 415.65,
+      "KW": 637.24,
+      "KY": 642.86,
+      "KZ": 801.79,
+      "LA": 232.12,
+      "LB": 369.47,
+      "LC": 650.0,
+      "LK": 509.78,
+      "LR": 435.9,
+      "LS": 20.83,
+      "LT": 139.34,
+      "LU": 134.62,
+      "LV": 136.22,
+      "LY": 830.53,
+      "MA": 577.65,
+      "MD": 629.56,
+      "ME": 413.51,
+      "MG": 477.27,
+      "MK": 568.97,
+      "ML": 394.5,
+      "MM": 569.69,
+      "MN": 784.01,
+      "MO": 448.98,
+      "MQ": 516.78,
+      "MR": 481.71,
       "MS": 1000.0,
-      "MT": 500.0,
-      "MU": 631.94,
-      "MV": 651.52,
-      "MW": 133.8,
-      "MX": 515.62,
-      "MY": 605.83,
-      "MZ": 150.25,
-      "NA": 70.06,
-      "NC": 610.12,
-      "NE": 622.22,
-      "NG": 513.24,
-      "NI": 352.05,
-      "NL": 324.99,
-      "NO": 26.5,
-      "NP": 24.51,
+      "MT": 484.16,
+      "MU": 633.03,
+      "MV": 611.77,
+      "MW": 54.65,
+      "MX": 483.14,
+      "MY": 604.43,
+      "MZ": 127.81,
+      "NA": 47.62,
+      "NC": 585.76,
+      "NE": 687.5,
+      "NG": 507.85,
+      "NI": 288.33,
+      "NL": 252.7,
+      "NO": 30.75,
+      "NP": 23.36,
       "NR": 750.0,
-      "NZ": 117.46,
-      "OM": 598.27,
-      "PA": 175.0,
-      "PE": 312.59,
-      "PF": 471.43,
-      "PG": 526.75,
-      "PH": 641.96,
-      "PK": 435.89,
-      "PL": 733.18,
-      "PM": 800.0,
-      "PR": 719.85,
-      "PS": 511.63,
-      "PT": 225.19,
-      "PY": 25.49,
-      "QA": 642.01,
-      "RE": 519.03,
-      "RO": 297.94,
-      "RS": 695.69,
-      "RU": 451.95,
-      "RW": 294.12,
-      "SA": 706.79,
-      "SB": 727.27,
-      "SC": 615.39,
-      "SD": 288.13,
-      "SE": 41.07,
-      "SG": 505.22,
-      "SI": 257.49,
-      "SK": 142.59,
+      "NZ": 120.11,
+      "OM": 545.33,
+      "PA": 258.74,
+      "PE": 263.27,
+      "PF": 436.62,
+      "PG": 513.74,
+      "PH": 613.38,
+      "PK": 398.61,
+      "PL": 614.98,
+      "PM": 600.0,
+      "PR": 664.53,
+      "PS": 460.78,
+      "PT": 112.29,
+      "PY": 24.86,
+      "QA": 602.83,
+      "RE": 525.22,
+      "RO": 245.55,
+      "RS": 670.8,
+      "RU": 446.17,
+      "RW": 301.89,
+      "SA": 691.95,
+      "SB": 636.36,
+      "SC": 571.43,
+      "SD": 214.33,
+      "SE": 35.82,
+      "SG": 498.74,
+      "SI": 227.65,
+      "SK": 96.49,
       "SL": 47.62,
-      "SN": 685.05,
-      "SO": 634.15,
-      "SR": 356.44,
-      "SS": 684.21,
-      "ST": 600.0,
-      "SV": 211.29,
-      "SY": 758.35,
-      "SZ": 189.19,
-      "TC": 703.7,
-      "TD": 677.42,
-      "TG": 492.06,
-      "TH": 599.02,
-      "TJ": 99.34,
-      "TM": 1389.72,
-      "TN": 592.21,
-      "TO": 625.0,
-      "TR": 453.01,
-      "TT": 723.94,
-      "TW": 678.87,
-      "TZ": 374.08,
-      "UA": 271.41,
-      "UG": 52.27,
-      "US": 410.37,
-      "UY": 149.49,
-      "UZ": 1176.69,
-      "VC": 500.0,
-      "VE": 298.5,
-      "VG": 714.29,
-      "VI": 685.71,
-      "VN": 409.8,
-      "VU": 571.43,
-      "WS": 470.59,
-      "XK": 945.01,
-      "YE": 613.64,
-      "ZA": 762.46,
-      "ZM": 86.39,
-      "ZW": 433.38
+      "SN": 535.4,
+      "SO": 523.81,
+      "SR": 383.18,
+      "SS": 610.17,
+      "ST": 555.56,
+      "SV": 99.29,
+      "SY": 682.27,
+      "SZ": 142.86,
+      "TC": 653.85,
+      "TD": 615.39,
+      "TG": 478.26,
+      "TH": 555.43,
+      "TJ": 98.7,
+      "TM": 1306.3,
+      "TN": 560.25,
+      "TO": 571.43,
+      "TR": 469.7,
+      "TT": 682.11,
+      "TW": 635.15,
+      "TZ": 371.59,
+      "UA": 250.47,
+      "UG": 57.39,
+      "US": 383.55,
+      "UY": 96.7,
+      "UZ": 1121.18,
+      "VC": 600.0,
+      "VE": 180.25,
+      "VG": 647.06,
+      "VI": 641.79,
+      "VN": 486.13,
+      "VU": 500.0,
+      "WS": 400.0,
+      "XK": 958.72,
+      "YE": 586.32,
+      "ZA": 713.48,
+      "ZM": 111.0,
+      "ZW": 298.44
     }
 
     # @see {co2.js/data/output/marginal-intensities-2021.json}