Update to fix missing region for observations with no measurement

scripts/map_prodcom_bulk_sold.dlog

@@ -9,7 +9,7 @@
         :-
         PRODCOM_DATA_NEW(?Decl, ?Year, ?PRCCODE, ?Indicators, ?ObsValue, ?ObsFlag),
         FILTER(?Indicators = "PRODQNT" || (?Indicators = "PQNTFLAG" && ?ObsValue = ":C")),
-        BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, "PRODQNT")))) AS ?ID),
+        BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, "PRODQNT_SOLD")))) AS ?ID),
         BIND(IRI(CONCAT(STR(ufpc2017:), "Object-", ?PRCCODE)) AS ?Object) ,
         BIND(IRI(CONCAT(STR(:), "TimePeriod_YearOf", ?Year)) AS ?TimePeriod) .
 
@@ -17,17 +17,19 @@
 [?ID, :hasRegion, ?Region]
        :-
        PRODCOM_DATA_NEW(?Decl, ?Year, ?PRCCODE, ?Indicators, ?ObsValue, ?ObsFlag),
+       FILTER(?Indicators = "PRODQNT" || (?Indicators = "PQNTFLAG" && ?ObsValue = ":C")),
        GEONAMES(?Decl, ?RegionID) ,
        FILTER(?RegionID != "0"),
-       BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, ?Indicators)))) AS ?ID),
+       BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, "PRODQNT_SOLD")))) AS ?ID),
        BIND(IRI(CONCAT(STR(gnd:), STR(?RegionID))) AS ?Region) .
 
 [?ID, :hasRegion, ?Region]
        :-
        PRODCOM_DATA_NEW(?Decl, ?Year, ?PRCCODE, ?Indicators, ?ObsValue, ?ObsFlag),
+       FILTER(?Indicators = "PRODQNT" || (?Indicators = "PQNTFLAG" && ?ObsValue = ":C")),
        GEONAMES(?Decl, ?RegionID) ,
        FILTER(?RegionID = "0"),
-       BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, ?Indicators)))) AS ?ID),
+       BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, "PRODQNT_SOLD")))) AS ?ID),
        BIND(IRI(CONCAT(STR(ufpc:), "Region-", STR(?Decl))) AS ?Region) .
 
 
@@ -37,7 +39,7 @@
         :-
         PRODCOM_DATA_NEW(?Decl, ?Year, ?PRCCODE, ?Indicators, ?ObsValue, ?ObsFlag),
         FILTER(?Indicators = "PRODQNT"),
-        BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, ?Indicators)))) AS ?ID) ,
+        BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, "PRODQNT_SOLD")))) AS ?ID) ,
         :DirectObservation[?ID] ,
 
         # Don't use the measurement value when there is a "confidential" flag,
@@ -51,7 +53,7 @@
         :-
         PRODCOM_DATA_NEW(?Decl, ?Year, ?PRCCODE, ?Indicators, ?ObsValue, ?ObsFlag),
         FILTER(?Indicators = "QNTUNIT"),
-        BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, "PRODQNT")))) AS ?ID) ,
+        BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, "PRODQNT_SOLD")))) AS ?ID) ,
         :DirectObservation[?ID] ,
         BIND(IRI(CONCAT(STR(:), "Unit-", SHA256(?ObsValue))) AS ?UnitID) .
 
@@ -60,7 +62,7 @@
         :-
         PRODCOM_DATA_NEW(?Decl, ?Year, ?PRCCODE, ?Indicators, ?ObsValue, ?ObsFlag),
         FILTER(?Indicators = "PQNTFLAG"),
-        BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, "PRODQNT")))) AS ?ID) ,
+        BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, "PRODQNT_SOLD")))) AS ?ID) ,
         :DirectObservation[?ID] .
 
 

scripts/map_prodcom_bulk_total.dlog

@@ -9,7 +9,7 @@
         :-
         PRODCOM_DATA_NEW(?Decl, ?Year, ?PRCCODE, ?Indicators, ?ObsValue, ?ObsFlag),
         FILTER(?Indicators = "PRODQNT" || (?Indicators = "PQNTFLAG" && ?ObsValue = ":C")),
-        BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, "PRODQNT")))) AS ?ID),
+        BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, "PRODQNT_TOTAL")))) AS ?ID),
         BIND(IRI(CONCAT(STR(ufpc2017:), "Object-", ?PRCCODE)) AS ?Object) ,
         BIND(IRI(CONCAT(STR(:), "TimePeriod_YearOf", ?Year)) AS ?TimePeriod) .
 
@@ -17,17 +17,19 @@
 [?ID, :hasRegion, ?Region]
        :-
        PRODCOM_DATA_NEW(?Decl, ?Year, ?PRCCODE, ?Indicators, ?ObsValue, ?ObsFlag),
+       FILTER(?Indicators = "PRODQNT" || (?Indicators = "PQNTFLAG" && ?ObsValue = ":C")),
        GEONAMES(?Decl, ?RegionID) ,
        FILTER(?RegionID != "0"),
-       BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, ?Indicators)))) AS ?ID),
+       BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, "PRODQNT_TOTAL")))) AS ?ID),
        BIND(IRI(CONCAT(STR(gnd:), STR(?RegionID))) AS ?Region) .
 
 [?ID, :hasRegion, ?Region]
        :-
        PRODCOM_DATA_NEW(?Decl, ?Year, ?PRCCODE, ?Indicators, ?ObsValue, ?ObsFlag),
+       FILTER(?Indicators = "PRODQNT" || (?Indicators = "PQNTFLAG" && ?ObsValue = ":C")),
        GEONAMES(?Decl, ?RegionID) ,
        FILTER(?RegionID = "0"),
-       BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, ?Indicators)))) AS ?ID),
+       BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, "PRODQNT_TOTAL")))) AS ?ID),
        BIND(IRI(CONCAT(STR(ufpc:), "Region-", STR(?Decl))) AS ?Region) .
 
 
@@ -37,7 +39,7 @@
         :-
         PRODCOM_DATA_NEW(?Decl, ?Year, ?PRCCODE, ?Indicators, ?ObsValue, ?ObsFlag),
         FILTER(?Indicators = "PRODQNT"),
-        BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, ?Indicators)))) AS ?ID) ,
+        BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, "PRODQNT_TOTAL")))) AS ?ID) ,
         :DirectObservation[?ID] ,
 
         # Don't use the measurement value when there is a "confidential" flag,
@@ -51,7 +53,7 @@
         :-
         PRODCOM_DATA_NEW(?Decl, ?Year, ?PRCCODE, ?Indicators, ?ObsValue, ?ObsFlag),
         FILTER(?Indicators = "QNTUNIT"),
-        BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, "PRODQNT")))) AS ?ID) ,
+        BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, "PRODQNT_TOTAL")))) AS ?ID) ,
         :DirectObservation[?ID] ,
         BIND(IRI(CONCAT(STR(:), "Unit-", SHA256(?ObsValue))) AS ?UnitID) .
 
@@ -60,7 +62,7 @@
         :-
         PRODCOM_DATA_NEW(?Decl, ?Year, ?PRCCODE, ?Indicators, ?ObsValue, ?ObsFlag),
         FILTER(?Indicators = "PQNTFLAG"),
-        BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, "PRODQNT")))) AS ?ID) ,
+        BIND(IRI(CONCAT(STR(ufpcd:), ?Year, "/Observation-", SHA256(CONCAT(?Decl, ?Year, ?PRCCODE, "PRODQNT_TOTAL")))) AS ?ID) ,
         :DirectObservation[?ID] .
 
 

tests/test_prodcom_bulk_sold.py

@@ -55,7 +55,7 @@ class TestProdcom2016Data:
         ("20141325", QUANTITYKIND.Mass),               # No PRDQNT, :C for PQNTFLAG
     ])
     def test_missing_measurements(self, rdfox, code, metric):
-        # these codes have no data for PRDQNT
+        # these codes have confidential PRDQNT (observation created with no measurement)
         result = rdfox.get_observations(
             PROBS.TimePeriod_YearOf2016,
             GEONAMES["2635167"] ,
@@ -63,14 +63,14 @@ class TestProdcom2016Data:
             PROBS.SoldProduction,
             object_code=code,
         )
-        assert_no_result(result)
+        assert_no_measurement(result)
 
     @pytest.mark.parametrize("code,metric", [
         ("10841270", QUANTITYKIND.Volume),
         ("20132455", QUANTITYKIND.Mass),
     ])
     def test_no_result_with_wrong_metric(self, rdfox, code, metric):
-        # same as above, but with the wrong metric
+        # query using wrong metric
         result = rdfox.get_observations(
             PROBS.TimePeriod_YearOf2016,
             GEONAMES["2635167"] ,

tests/test_prodcom_bulk_total.py

@@ -55,7 +55,7 @@ class TestProdcom2016Data:
         ("20141325", QUANTITYKIND.Mass),              # No PRDQNT, :C for PQNTFLAG
     ])
     def test_missing_measurements(self, rdfox, code, metric):
-        # these codes have no data for PRDQNT
+        # these codes have confidential PRDQNT (observation created with no measurement)
         result = rdfox.get_observations(
             PROBS.TimePeriod_YearOf2016,
             GEONAMES["2635167"] ,
@@ -63,14 +63,14 @@ class TestProdcom2016Data:
             PROBS.TotalProduction,
             object_code=code,
         )
-        assert_no_result(result)
+        assert_no_measurement(result)
 
     @pytest.mark.parametrize("code,metric", [
-        ("13201230", QUANTITYKIND.Mass),
-        ("20144370", QUANTITYKIND.AmountOfSubstance),
+        ("20132455", QUANTITYKIND.Mass),
+        ("20141325", QUANTITYKIND.AmountOfSubstance),
     ])
     def test_no_result_with_wrong_metric(self, rdfox, code, metric):
-        # same as above, but with the wrong metric
+        # query using wrong metric
         result = rdfox.get_observations(
             PROBS.TimePeriod_YearOf2016,
             GEONAMES["2635167"] ,