test: forgot about these...

access_logd

@@ -7,5 +7,5 @@ test -n "$ACCESS_LOGS_FLAGS" || exit 1
 umask 007
 
 # Read from fifo and load into database
-daemonize -p /run/access_logd.pid -u nobody -g www-data \
+daemonize -p /run/access_logd.pid -u nobody \
   /usr/bin/access_log $ACCESS_LOGS_FLAGS

shard.yml

@@ -1,5 +1,5 @@
 name: "access_log"
-version: "0.5.0"
+version: "0.5.6"
 authors:
 - "f <f@sutty.nl>"
 targets:

spec/swd_spec.cr

@@ -111,31 +111,31 @@ describe SWD do
 
   describe "#consumer_device_co2" do
     it "should extract the consumer device co2 emissions" do
-      swd.consumer_device_co2.should(eq 0.6406335995061729)
+      swd.consumer_device_co2.should(eq 0.6343432152938272)
     end
   end
 
   describe "#production_co2" do
     it "should extract the production co2 emissions" do
-      swd.production_co2.should(eq 0.23407766135802469)
+      swd.production_co2.should(eq 0.2317792517419753)
     end
   end
 
   describe "#network_co2" do
     it "should extract the network co2 emissions" do
-      swd.network_co2.should(eq 0.17247827679012348)
+      swd.network_co2.should(eq 0.1707847118098766)
     end
   end
 
   describe "#datacenter_co2" do
     it "should extract the datacenter co2 emissions" do
-      swd.datacenter_co2.should(eq 0.1847981537037037)
+      swd.datacenter_co2.should(eq 0.1829836197962963)
     end
   end
 
   describe "#total_co2" do
     it "should extract the total co2 emissions" do
-      swd.total_co2.should(eq 1.2319876913580248)
+      swd.total_co2.should(eq 1.2198907986419754)
     end
   end
 end

src/access_log.cr

@@ -1,3 +1,4 @@
+require "log"
 require "file_utils"
 require "json"
 require "socket"
@@ -5,11 +6,14 @@ require "sqlite3"
 require "pg"
 require "option_parser"
 require "uuid"
+require "system"
 require "./models/access_log"
 require "./models/crawler"
 require "./swd"
 
-VERSION = "0.5.0"
+VERSION = "0.5.6"
+
+Log.setup_from_env
 
 # Default socket location
 socket = "/tmp/access_log.socket"
@@ -21,7 +25,7 @@ crawler = false
 crawlers = [] of Crawler
 # Fields in database
 # TODO: Obtain them from AccessLog
-fields = %w[id remote_user host msec server_protocol request_method request_completion uri request_uri query_string status sent_http_content_type sent_http_content_encoding sent_http_etag sent_http_last_modified http_accept http_accept_encoding http_accept_language http_pragma http_cache_control http_if_none_match http_dnt http_user_agent http_origin http_referer request_time bytes_sent body_bytes_sent request_length http_connection pipe connection_requests geoip2_data_country_name geoip2_data_city_name ssl_server_name ssl_protocol ssl_early_data ssl_session_reused ssl_curves ssl_ciphers ssl_cipher sent_http_x_xss_protection sent_http_x_frame_options sent_http_x_content_type_options sent_http_strict_transport_security nginx_version pid crawler datacenter_co2 network_co2 consumer_device_co2 production_co2 total_co2]
+fields = %w[id remote_user host msec server_protocol request_method request_completion uri request_uri query_string status sent_http_content_type sent_http_content_encoding sent_http_etag sent_http_last_modified http_accept http_accept_encoding http_accept_language http_pragma http_cache_control http_if_none_match http_dnt http_user_agent http_origin http_referer request_time bytes_sent body_bytes_sent request_length http_connection pipe connection_requests geoip2_data_country_name geoip2_data_city_name ssl_server_name ssl_protocol ssl_early_data ssl_session_reused ssl_curves ssl_ciphers ssl_cipher sent_http_x_xss_protection sent_http_x_frame_options sent_http_x_content_type_options sent_http_strict_transport_security nginx_version pid crawler datacenter_co2 network_co2 consumer_device_co2 production_co2 total_co2 node]
 # Params for the query
 params = [] of String
 # SWD
@@ -31,6 +35,7 @@ datacenter : String? = nil
 renewable = false
 device_country = false
 intensity = :marginal
+node = false
 
 # Parse CLI flags
 OptionParser.parse do |p|
@@ -79,10 +84,14 @@ OptionParser.parse do |p|
   p.on "-A", "--average-intensity", "Use average intensity figures rather than marginal" do
     intensity = :average
   end
+
+  p.on "-n", "--node", "Send node hostname" do
+    node = true
+  end
 end
 
 # Parameterize values according to database URI
-if database.starts_with? "postgresql://"
+if database.starts_with?("postgresql://") || database.starts_with?("postgres://")
   params = fields.map_with_index do |_, i|
     "$#{i + 1}"
   end
@@ -121,7 +130,8 @@ db = DB.open database do |db|
   while true
     begin
       msg, _ = server.receive(1024 * 64) # 64K
-      _, _, _, _, json = msg.split(" ", 5)
+      _, json = msg.split("{", 2)
+      json = "{#{json}"
 
       # Parse input
       access_log = AccessLog.from_json(json || "{}")
@@ -184,10 +194,12 @@ db = DB.open database do |db|
               (swd ? s.try(&.network_co2) : nil),
               (swd ? s.try(&.consumer_device_co2) : nil),
               (swd ? s.try(&.production_co2) : nil),
-              (swd ? s.try(&.total_co2) : nil)
+              (swd ? s.try(&.total_co2) : nil),
+              (node ? System.hostname : nil)
 
     # Ignore parsing errors
-    rescue JSON::ParseException
+    rescue e : JSON::ParseException
+      Log.warn &.emit("Parse exception", error: e.message)
     rescue IO::Error
       server.close
       FileUtils.rm(socket) if File.exists? socket

src/average_intensities_by_country.cr

@@ -1,14 +1,29 @@
 require "json"
+require "csv"
 
-average_intensities = Hash(String, Float32).new
+csv = CSV.new(File.open("./countries_codes_and_coordinates.csv"), headers: true)
+countries = Hash(String, String).new
+
+while csv.next
+  countries[csv["Alpha-3 code"]] = csv["Alpha-2 code"]
+end
+
+average_intensities = Hash(String, Float64).new
 
 JSON.parse(ARGF.gets_to_end).as_h.each do |country, data|
-  if data["country_code_iso_2"].as_s.empty?
+  if data["country_code"].as_s.empty?
     STDERR.puts "not ok - Skipping #{country} because of missing ISO code"
     next
   end
 
-  average_intensities[data["country_code_iso_2"].as_s] = data["emissions_intensity_gco2_per_kwh"].as_s.to_f32
+  unless countries.has_key? data["country_code"].as_s
+    STDERR.puts "not ok - Skipping #{country} because of missing ISO2 code"
+    next
+  end
+
+  STDERR.puts "ok - #{country}"
+
+  average_intensities[countries[data["country_code"].as_s]] = data["emissions_intensity_gco2_per_kwh"].as_f
 end
 
 puts average_intensities.to_json

src/swd.cr

@@ -20,13 +20,14 @@ class SWD
   getter consumer_device_iso_code : String?
   getter intensity_calculator : Symbol
 
+  # @see {co2.js/blob/main/src/constants/index.js}
   GIGABYTE = 1000.0 * 1000.0 * 1000.0
   KWH_PER_GB = 0.81
   END_USER_DEVICE_ENERGY = 0.52
   NETWORK_ENERGY = 0.14
   DATACENTER_ENERGY = 0.15
   PRODUCTION_ENERGY = 0.19
-  GLOBAL_GRID_INTENSITY = 442.0
+  GLOBAL_GRID_INTENSITY = 437.66
   RENEWABLES_GRID_INTENSITY = 50.0
 
   @transfered_bytes_to_gb : Float32? | Float64? = nil

src/swd/intensity_data.cr

@@ -1,7 +1,217 @@
 class SWD
   module IntensityData
-    # @see {co2.js/data/output/average-intensities-2021.json}
-    AVERAGE_INTENSITY_BY_ISO_CODE = {"AR": 365.292, "AM": 206.522, "AU": 526.876, "AT": 145.083, "AZ": 536.585, "BD": 559.606, "BY": 472.727, "BE": 156.063, "BO": 311.475, "BA": 470.982, "BR": 144.677, "BG": 364.136, "BI": 275.862, "CA": 123.859, "CL": 395.565, "CN": 549.288, "CR": 30.903, "HR": 212.161, "CY": 601.19, "CZ": 401.272, "DK": 240.419, "EC": 132.964, "EG": 470.879, "SV": 180.87, "EE": 488.529, "FI": 152.651, "FR": 67.781, "GE": 105.685, "DE": 363.982, "GR": 363.388, "HU": 236.271, "IN": 632.656, "IE": 361.274, "IT": 340.937, "JP": 460.647, "KZ": 656.097, "KE": 104.0, "LV": 226.351, "LT": 247.475, "LU": 183.824, "MT": 452.055, "MX": 391.582, "MD": 642.512, "MN": 725.26, "ME": 335.958, "NL": 386.189, "MK": 444.191, "NO": 26.131, "PK": 363.065, "PE": 241.492, "PH": 579.689, "PL": 657.138, "PT": 222.632, "RO": 255.718, "RU": 355.431, "SA": 568.967, "SN": 540.098, "RS": 549.083, "SG": 488.21, "SK": 173.854, "SI": 241.956, "ZA": 706.991, "KR": 442.389, "ES": 193.737, "SE": 43.9, "CH": 58.952, "TW": 565.629, "TJ": 72.823, "TH": 503.034, "TN": 470.848, "TR": 432.293, "UA": 240.28, "GB": 268.255, "US": 378.625, "VN": 491.192}
+    # @see {co2.js/data/output/average-intensities.json}
+    AVERAGE_INTENSITY_BY_ISO_CODE = {
+      "AE": 407.98,
+      "AF": 120.48,
+      "AG": 657.14,
+      "AL": 23.44,
+      "AM": 222.68,
+      "AO": 195.98,
+      "AR": 344.31,
+      "AS": 687.5,
+      "AT": 158.22,
+      "AU": 501.7,
+      "AW": 591.4,
+      "AZ": 469.58,
+      "BA": 553.47,
+      "BB": 644.86,
+      "BD": 574.28,
+      "BE": 167.11,
+      "BF": 611.43,
+      "BG": 399.72,
+      "BH": 494.02,
+      "BI": 250.0,
+      "BJ": 666.67,
+      "BN": 493.59,
+      "BO": 335.4,
+      "BR": 102.04,
+      "BS": 698.11,
+      "BT": 24.44,
+      "BW": 794.52,
+      "BY": 425.9,
+      "BZ": 484.38,
+      "CA": 125.84,
+      "CD": 25.36,
+      "CF": 0.0,
+      "CG": 395.52,
+      "CH": 41.28,
+      "CI": 410.75,
+      "CK": 400.0,
+      "CL": 332.61,
+      "CM": 278.26,
+      "CN": 533.98,
+      "CO": 163.99,
+      "CR": 37.21,
+      "CU": 654.68,
+      "CV": 600.0,
+      "CY": 589.35,
+      "CZ": 414.8,
+      "DE": 385.39,
+      "DJ": 666.67,
+      "DK": 180.42,
+      "DM": 529.41,
+      "DO": 549.8,
+      "DZ": 485.49,
+      "EC": 183.63,
+      "EE": 460.26,
+      "EG": 469.63,
+      "ER": 688.89,
+      "ES": 217.42,
+      "ET": 25.19,
+      "FI": 131.71,
+      "FJ": 289.47,
+      "FK": 500.0,
+      "FO": 428.57,
+      "FR": 84.88,
+      "GA": 397.38,
+      "GB": 261.16,
+      "GD": 714.29,
+      "GE": 134.83,
+      "GF": 254.72,
+      "GH": 361.2,
+      "GL": 133.33,
+      "GM": 700.0,
+      "GN": 208.63,
+      "GP": 623.53,
+      "GQ": 492.96,
+      "GR": 344.41,
+      "GT": 304.71,
+      "GU": 670.33,
+      "GW": 750.0,
+      "GY": 642.28,
+      "HK": 609.93,
+      "HN": 373.96,
+      "HR": 246.29,
+      "HT": 606.06,
+      "HU": 222.1,
+      "ID": 619.03,
+      "IE": 346.43,
+      "IL": 537.57,
+      "IN": 633.4,
+      "IQ": 531.36,
+      "IR": 487.86,
+      "IS": 28.56,
+      "IT": 372.63,
+      "JM": 537.93,
+      "JO": 391.13,
+      "JP": 494.86,
+      "KE": 101.13,
+      "KG": 104.43,
+      "KH": 400.46,
+      "KI": 666.67,
+      "KM": 714.29,
+      "KN": 681.82,
+      "KP": 102.42,
+      "KR": 437.6,
+      "KW": 574.56,
+      "KY": 684.93,
+      "KZ": 635.57,
+      "LA": 242.18,
+      "LB": 663.1,
+      "LC": 685.71,
+      "LK": 501.53,
+      "LR": 304.35,
+      "LS": 20.0,
+      "LT": 195.7,
+      "LU": 162.6,
+      "LV": 183.43,
+      "LY": 558.85,
+      "MA": 630.75,
+      "MD": 666.67,
+      "ME": 392.75,
+      "MG": 483.25,
+      "MK": 543.71,
+      "ML": 463.13,
+      "MM": 330.8,
+      "MN": 749.66,
+      "MO": 491.53,
+      "MQ": 698.63,
+      "MR": 526.6,
+      "MS": 1000.0,
+      "MT": 433.48,
+      "MU": 611.11,
+      "MV": 651.52,
+      "MW": 133.8,
+      "MX": 423.81,
+      "MY": 543.87,
+      "MZ": 126.63,
+      "NA": 63.69,
+      "NC": 610.12,
+      "NE": 622.22,
+      "NG": 368.11,
+      "NI": 354.21,
+      "NL": 354.31,
+      "NO": 28.93,
+      "NP": 24.51,
+      "NR": 750.0,
+      "NZ": 105.22,
+      "OM": 488.27,
+      "PA": 152.68,
+      "PE": 256.51,
+      "PF": 471.43,
+      "PG": 526.75,
+      "PH": 594.45,
+      "PK": 344.16,
+      "PL": 633.23,
+      "PM": 800.0,
+      "PR": 612.39,
+      "PS": 465.12,
+      "PT": 234.61,
+      "PY": 25.49,
+      "QA": 490.28,
+      "RE": 519.03,
+      "RO": 264.24,
+      "RS": 582.13,
+      "RU": 363.68,
+      "RW": 294.12,
+      "SA": 557.78,
+      "SB": 727.27,
+      "SC": 615.39,
+      "SD": 288.13,
+      "SE": 45.12,
+      "SG": 488.78,
+      "SI": 237.38,
+      "SK": 140.14,
+      "SL": 47.62,
+      "SN": 523.13,
+      "SO": 634.15,
+      "SR": 356.44,
+      "SS": 684.21,
+      "ST": 600.0,
+      "SV": 194.23,
+      "SY": 541.17,
+      "SZ": 189.19,
+      "TC": 703.7,
+      "TD": 677.42,
+      "TG": 460.32,
+      "TH": 501.57,
+      "TJ": 83.63,
+      "TM": 490.19,
+      "TN": 469.43,
+      "TO": 625.0,
+      "TR": 413.6,
+      "TT": 491.41,
+      "TW": 560.98,
+      "TZ": 366.75,
+      "UA": 232.74,
+      "UG": 52.27,
+      "US": 368.1,
+      "UY": 150.13,
+      "UZ": 505.41,
+      "VC": 500.0,
+      "VE": 212.48,
+      "VG": 714.29,
+      "VI": 685.71,
+      "VN": 386.49,
+      "VU": 571.43,
+      "WS": 470.59,
+      "XK": 767.0,
+      "YE": 559.66,
+      "ZA": 708.21,
+      "ZM": 84.7,
+      "ZW": 392.28
+    }
 
     # @see {co2.js/data/output/marginal-intensities-2021.json}
     MARGINAL_INTENSITY_BY_ISO_CODE = {"AF": 414.0, "AL": 0.0, "DZ": 528.0, "AS": 753.0, "AD": 188.0, "AO": 1476.0, "AI": 753.0, "AG": 753.0, "AR": 478.0, "AM": 390.0, "AW": 753.0, "AU": 808.0, "AT": 242.0, "AZ": 534.0, "BS": 753.0, "BH": 726.0, "BD": 528.0, "BB": 749.0, "BY": 400.0, "BE": 252.0, "BZ": 403.0, "BJ": 745.0, "BM": 753.0, "BT": 0.0, "BO": 604.0, "BQ": 753.0, "BA": 1197.0, "BW": 1486.0, "BR": 284.0, "VG": 753.0, "BN": 681.0, "BG": 911.0, "BF": 753.0, "BI": 414.0, "KH": 1046.0, "CM": 659.0, "CA": 372.0, "KY": 753.0, "CV": 753.0, "CY": 751.0, "CF": 188.0, "TD": 753.0, "CL": 657.0, "CN": 899.0, "CO": 410.0, "KM": 753.0, "CD": 0.0, "CG": 659.0, "CK": 753.0, "CR": 108.0, "CI": 466.0, "HR": 294.0, "CU": 559.0, "CW": 876.0, "CZ": 902.0, "DK": 362.0, "DJ": 753.0, "DM": 753.0, "DO": 601.0, "EC": 560.0, "EG": 554.0, "SV": 547.0, "GQ": 632.0, "ER": 915.0, "EE": 1057.0, "SZ": 0.0, "ET": 0.0, "FK": 753.0, "FO": 753.0, "FJ": 640.0, "FI": 267.0, "FR": 158.0, "GF": 423.0, "PF": 753.0, "GA": 946.0, "GM": 753.0, "GE": 289.0, "DE": 650.0, "GH": 495.0, "GI": 779.0, "GR": 507.0, "GL": 264.0, "GD": 753.0, "GP": 753.0, "GU": 753.0, "GT": 798.0, "GN": 753.0, "GW": 753.0, "GY": 847.0, "HT": 1048.0, "HN": 662.0, "HU": 296.0, "IS": 0.0, "IN": 951.0, "ID": 783.0, "IR": 592.0, "IQ": 1080.0, "IE": 380.0, "IM": 436.0, "IL": 394.0, "IT": 414.0, "JM": 711.0, "JP": 471.0, "JO": 529.0, "KZ": 797.0, "KE": 574.0, "KI": 753.0, "KP": 754.0, "KR": 555.0, "XK": 1145.0, "KW": 675.0, "KG": 217.0, "LA": 1069.0, "LV": 240.0, "LB": 794.0, "LS": 0.0, "LR": 677.0, "LY": 668.0, "LI": 151.0, "LT": 211.0, "LU": 220.0, "MG": 876.0, "MW": 489.0, "MY": 551.0, "MV": 753.0, "ML": 1076.0, "MT": 520.0, "MH": 753.0, "MQ": 753.0, "MR": 753.0, "MU": 700.0, "YT": 753.0, "MX": 531.0, "FM": 753.0, "MD": 541.0, "MC": 158.0, "MN": 1366.0, "ME": 899.0, "MS": 753.0, "MA": 729.0, "MZ": 234.0, "MM": 719.0, "NA": 355.0, "NR": 753.0, "NP": 0.0, "NL": 326.0, "NC": 779.0, "NZ": 246.0, "NI": 675.0, "NE": 772.0, "NG": 526.0, "NU": 753.0, "MK": 851.0, "MP": 753.0, "NO": 47.0, "OM": 479.0, "PK": 592.0, "PW": 753.0, "PS": 719.0, "PA": 477.0, "PG": 597.0, "PY": 0.0, "PE": 473.0, "PH": 672.0, "PL": 828.0, "PT": 389.0, "PR": 596.0, "QA": 503.0, "RE": 772.0, "RO": 489.0, "RU": 476.0, "RW": 712.0, "SH": 753.0, "KN": 753.0, "LC": 753.0, "MF": 753.0, "PM": 753.0, "VC": 753.0, "WS": 753.0, "SM": 414.0, "ST": 753.0, "SA": 592.0, "SN": 870.0, "RS": 1086.0, "SC": 753.0, "SL": 489.0, "SG": 379.0, "SX": 753.0, "SK": 332.0, "SI": 620.0, "SB": 753.0, "SO": 753.0, "ZA": 1070.0, "SS": 890.0, "ES": 402.0, "LK": 731.0, "SD": 736.0, "SR": 1029.0, "SE": 68.0, "CH": 48.0, "SY": 713.0, "TW": 484.0, "TJ": 255.0, "TZ": 531.0, "TH": 450.0, "TL": 753.0, "TG": 859.0, "TO": 753.0, "TT": 559.0, "TN": 468.0, "TR": 376.0, "TM": 927.0, "TC": 753.0, "TV": 753.0, "UG": 279.0, "UA": 768.0, "AE": 556.0, "GB": 380.0, "US": 416.0, "UY": 174.0, "UZ": 612.0, "VU": 753.0, "VE": 711.0, "VN": 560.0, "VI": 650.0, "YE": 807.0, "ZM": 416.0, "ZW": 1575.0}
@@ -9,13 +219,13 @@ class SWD
     # Return the marginal intensity for a country when provided an ISO
     # code
     def self.marginal_by_iso(iso : String) : Float64?
-      MARGINAL_INTENSITY_BY_ISO_CODE[iso]
+      MARGINAL_INTENSITY_BY_ISO_CODE[iso] if MARGINAL_INTENSITY_BY_ISO_CODE.has_key? iso
     end
 
     # Return the average intensity for a country when provided an ISO
     # code
     def self.average_by_iso(iso : String) : Float64?
-      AVERAGE_INTENSITY_BY_ISO_CODE[iso]
+      AVERAGE_INTENSITY_BY_ISO_CODE[iso] if AVERAGE_INTENSITY_BY_ISO_CODE.has_key? iso
     end
 
     def self.by_iso(iso : String, type : Symbol = :marginal) : Float64?