PRODCOM data conversion testing

scripts/map_prodcom.dlog

@@ -6,6 +6,7 @@
 [?ID, :hasRole, :SoldProduction] ,
 [?ID, :partOfDataset, ?Dataset] ,
 [?ID, :bound, :ExactBound] ,
+[?ID, :metric, quantitykind:Mass] ,
 ufu:NG(?ID, ufu:unit, ?UnitID)
         :- ufrd:PRODCOM_DATA(?IDstring, ?PRCCODEstring, ?PRODQNT, ?QNTUNIT),
 

tests/conftest.py

@@ -0,0 +1,41 @@
+"""Common configuration shared by all tests in this directory."""
+
+
+import re
+from pathlib import Path
+import shutil
+import pytest
+
+
+from probs_runner import probs_endpoint
+
+
+ROOT = Path(__file__).parent.parent
+
+
+@pytest.fixture(scope="class")
+def tmp_path_class(request, tmp_path_factory):
+    """Like build-in tmp_path fixture but scoped to class."""
+    name = request.node.name
+    name = re.sub(r"[\W]", "_", name)
+    MAXVAL = 30
+    name = name[:MAXVAL]
+    return tmp_path_factory.mktemp(name, numbered=True)
+
+
+@pytest.fixture(scope="class")
+def rdfox(request, tmp_path_class):
+    """Return a function to run a set of datasources"""
+    working_dir = tmp_path_class
+    data_paths = [
+        ROOT / Path(p)
+        for p in request.cls.endpoint_data
+    ]
+
+    print("Starting endpoint with", data_paths)
+    with probs_endpoint(
+            data_paths, working_dir, port=12119
+    ) as rdfox:
+        yield rdfox
+    shutil.rmtree(working_dir)
+    print("Finished endpoint with", data_paths)

tests/test_prodcom.py

@@ -0,0 +1,419 @@
+#!/usr/bin/env python3
+import pytest
+from numpy import isnan
+from hashlib import sha256
+
+from rdflib import Namespace
+from probs_runner import PROBS, QUANTITYKIND
+from numpy.testing import assert_allclose
+
+PRODCOM2016 = Namespace("https://ukfires.org/probs/ontology/prodcom/2016/")
+PRODCOM2017 = Namespace("https://ukfires.org/probs/ontology/prodcom/2017/")
+GEONAMES = Namespace("https://sws.geonames.org/")
+
+def assert_exact_results(result, expected_value):
+    assert len(result) == 1
+    assert result[0].bound == PROBS.ExactBound
+    assert_allclose(result[0].measurement, expected_value, rtol=1e-3)
+
+
+def assert_none(result):
+    assert len(result) == 1
+    assert isnan(result[0].measurement)
+
+
+class TestProdcom2016Data:
+    """Test PRODCOM2016 DATA conversion"""
+
+    endpoint_data = ["outputs/PRODCOM2016DATA.nt.gz"]
+
+    def test_1(self, rdfox):
+        ob = sha256("10113250".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2016,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2016["Object-"+ob]
+        )
+        assert_exact_results(result, 2717950)
+
+
+    def test_2(self, rdfox):
+        ob = sha256("10841270".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2016,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2016["Object-"+ob],
+        )
+        assert_exact_results(result, 622435736)
+
+
+    def test_3(self, rdfox):
+        ob = sha256("14391090".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2016,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2016["Object-"+ob],
+        )
+        assert_none(result)
+
+
+    def test_4(self, rdfox):
+        ob = sha256("15115100".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2016,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2016["Object-"+ob],
+        )
+        assert_exact_results(result, 0)
+
+
+    def test_5(self, rdfox):
+        ob = sha256("17121439".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2016,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2016["Object-"+ob],
+        )
+        assert_exact_results(result, 43196000)
+
+
+    def test_6(self, rdfox):
+        ob = sha256("20132455".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2016,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2016["Object-"+ob],
+        )
+        assert_exact_results(result, 1396308)
+
+
+    def test_7(self, rdfox):
+        ob = sha256("20141325".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2016,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2016["Object-"+ob],
+        )
+        assert_none(result)
+
+
+    def test_8(self, rdfox):
+        ob = sha256("21103159".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2016,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2016["Object-"+ob],
+        )
+        assert_exact_results(result, 223404)
+
+
+    def test_9(self, rdfox):
+        ob = sha256("24421154".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2016,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2016["Object-"+ob],
+        )
+        assert_exact_results(result, 383555000)
+
+
+    def test_10(self, rdfox):
+        ob = sha256("25931230".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2016,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2016["Object-"+ob],
+        )
+        assert_exact_results(result, 6279000)
+
+
+class TestProdcom2017Data:
+    """Test PRODCOM2017 DATA conversion"""
+
+    endpoint_data = ["outputs/PRODCOM2017DATA.nt.gz"]
+
+    
+    def test_1(self, rdfox):
+        ob = sha256("10721253".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2017,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2017["Object-"+ob],
+        )
+        assert_exact_results(result, 244871494)
+
+
+    def test_2(self, rdfox):
+        ob = sha256("13941280".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2017,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2017["Object-"+ob],
+        )
+        assert_exact_results(result, 1591)
+
+
+    def test_3(self, rdfox):
+        ob = sha256("16212400".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2017,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2017["Object-"+ob],
+        )
+        assert_exact_results(result, 0)
+
+
+    def test_4(self, rdfox):
+        ob = sha256("20143235".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2017,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2017["Object-"+ob],
+        )
+        assert_exact_results(result, 13379855)
+
+
+    def test_5(self, rdfox):
+        ob = sha256("20421975".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2017,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2017["Object-"+ob],
+        )
+        assert_none(result)
+
+
+    def test_6(self, rdfox):
+        ob = sha256("23111290".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2017,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2017["Object-"+ob],
+        )
+        assert_exact_results(result, 28514781)
+
+
+    def test_7(self, rdfox):
+        ob = sha256("24442200".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2017,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2017["Object-"+ob],
+        )
+        assert_exact_results(result, 3374000)
+
+
+    def test_8(self, rdfox):
+        ob = sha256("25711430".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2017,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2017["Object-"+ob],
+        )
+        assert_exact_results(result, 329685)
+
+
+    def test_9(self, rdfox):
+        ob = sha256("28131380".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2017,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2017["Object-"+ob],
+        )
+        assert_exact_results(result, 149173)
+
+    def test_10(self, rdfox):
+        ob = sha256("29323040".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2017,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2017["Object-"+ob]
+        )
+        assert_exact_results(result, 54163804)
+
+    def test_11(self, rdfox):
+        ob = sha256("11021213".encode('utf-8')).hexdigest()
+        print(ob)
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2017,
+            GEONAMES["2635167"],
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2017["Object-"+ob]
+        )
+        assert_none(result) 
+
+class TestProdcom2018Data:
+    """Test PRODCOM2018 DATA conversion"""
+
+    endpoint_data = ["outputs/PRODCOM2018DATA.nt.gz"]
+
+    
+    def test_1(self, rdfox):
+        ob = sha256("10131120".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2018,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2017["Object-"+ob],
+        )
+        assert_exact_results(result, 104282828)
+
+
+    def test_2(self, rdfox):
+        ob = sha256("10392510".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2018,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2017["Object-"+ob],
+        )
+        assert_none(result)
+
+    def test_3(self, rdfox):
+        ob = sha256("11052000".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2018,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2017["Object-"+ob],
+        )
+        assert_exact_results(result, 737526732)
+
+
+    def test_4(self, rdfox):
+        ob = sha256("13921530".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2018,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2017["Object-"+ob],
+        )
+        assert_exact_results(result, 1197738)
+
+
+    def test_5(self, rdfox):
+        ob = sha256("15112200".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2018,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2017["Object-"+ob],
+        )
+        assert_exact_results(result, 0)
+
+
+    def test_6(self, rdfox):
+        ob = sha256("17111300".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2018,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2017["Object-"+ob],
+        )
+        assert_none(result)
+
+
+    def test_7(self, rdfox):
+        ob = sha256("17231330".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2018,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2017["Object-"+ob],
+        )
+        assert_exact_results(result, 13795961)
+
+
+    def test_8(self, rdfox):
+        ob = sha256("20133133".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2018,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2017["Object-"+ob],
+        )
+        assert_none(result)
+
+
+    def test_9(self, rdfox):
+        ob = sha256("20142333".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2018,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2017["Object-"+ob],
+        )
+        assert_exact_results(result, 0)
+
+    def test_10(self, rdfox):
+        ob = sha256("20165350".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2018,
+            GEONAMES["2635167"] ,
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2017["Object-"+ob]
+        )
+        assert_none(result)
+
+    def test_11(self, rdfox):
+        ob = sha256("22231470".encode('utf-8')).hexdigest()
+        result = rdfox.get_observations(
+            PROBS.TimePeriod_YearOf2018,
+            GEONAMES["2635167"],
+            QUANTITYKIND.Mass,
+            PROBS.SoldProduction,
+            object_=PRODCOM2017["Object-"+ob]
+        )
+        assert_exact_results(result, 308843000) 
+